netgrep 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 0379650c8ebcdb1a4e2e72db58fe1b1ada7e3886
4
+ data.tar.gz: b6e24cf116aede9c004872be2b9938a9d95530fc
5
+ SHA512:
6
+ metadata.gz: a9ef509196130027e388fd334cf41a032a8319c36d8e75ce4cb4ba25b7f66033cb5d49c71e69d5d8c855f09b6695b995054b8f1d0a34ab36157664600f06ba57
7
+ data.tar.gz: 5e081fb5a201420391c8ce2e84a77a16f589e8d4ceb4306c99b9d520b1f4eb08cf1dcb2a4de12167a25e474e996f96e2f9788bff2c395608435739337deafd0d
data/.gitignore ADDED
@@ -0,0 +1,11 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ /spec/fixtures/fsb-large.log
11
+ /.idea
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --format documentation
2
+ --color
data/.ruby-version ADDED
@@ -0,0 +1 @@
1
+ 2.2.3
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in netgrep.gemspec
4
+ gemspec
data/README.md ADDED
@@ -0,0 +1,36 @@
1
+ # Netgrep
2
+
3
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/netgrep`. To experiment with that code, run `bin/console` for an interactive prompt.
4
+
5
+ TODO: Delete this and the text above, and describe your gem
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'netgrep'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install netgrep
22
+
23
+ ## Usage
24
+
25
+ TODO: Write usage instructions here
26
+
27
+ ## Development
28
+
29
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
30
+
31
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
32
+
33
+ ## Contributing
34
+
35
+ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/netgrep.
36
+
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "netgrep"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
data/bin/netblocks ADDED
@@ -0,0 +1,43 @@
1
+ 2607:f140:1:8003:6a5b:35ff:fe85:90f4/64
2
+ 10.254.1.0/24
3
+ 10.254.10.0/24
4
+ 10.254.13.0/24
5
+ 10.254.15.0/24
6
+ 10.254.16.0/24
7
+ 10.254.18.0/23
8
+ 10.254.2.0/24
9
+ 10.254.20.0/24
10
+ 10.254.25.0/24
11
+ 10.254.26.0/24
12
+ 10.254.27.0/24
13
+ 10.254.28.0/24
14
+ 10.254.3.0/24
15
+ 10.254.4.0/24
16
+ 10.254.9.0/24
17
+ 10.35.1.0
18
+ 128.32.16.64/26
19
+ 128.32.169.128/25
20
+ 128.32.185.0/26
21
+ 128.32.189.0/24
22
+ 128.32.209.0/24
23
+ 128.32.249.0/24
24
+ 128.32.25.0/24
25
+ 128.32.31.0/24
26
+ 128.32.80.32/27
27
+ 169.229.131.0/25
28
+ 169.229.161.0/25
29
+ 169.229.164.0/24
30
+ 169.229.217.160/27
31
+ 169.229.217.192/27
32
+ 169.229.218.0/26
33
+ 169.229.219.0/26
34
+ 169.229.243.0/26
35
+ 169.229.56.128/26
36
+ 132.249.243.0/24
37
+ 192.31.161.0/24
38
+ 192.31.95.128/27
39
+ 192.31.95.160/27
40
+ 192.31.95.192/26
41
+ 192.31.95.96/27
42
+ 192.58.221.0/25
43
+ 192.58.221.128/25
data/bin/netgrep ADDED
@@ -0,0 +1,21 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Exit cleanly from an early interrupt
4
+ Signal.trap("INT") { exit 1 }
5
+
6
+ require "bundler/setup"
7
+ require "netgrep"
8
+
9
+ begin
10
+ opts = Netgrep::Options.parse(ARGV)
11
+ runner = Netgrep::Runner.new(
12
+ log_file: opts.log_file,
13
+ netblocks_file: opts.netblocks_file,
14
+ num_workers: opts.num_workers
15
+ )
16
+ runner.run
17
+ rescue Errno::ENOENT => err
18
+ abort "netgrep: #{err.message}"
19
+ rescue OptionParser::InvalidOption, OptionParser::InvalidArgument
20
+ abort "usage: netgrep [-w NUM_WORKERS] -f [NETBLOCKS_FILE] [LOG_FILE]"
21
+ end
data/bin/setup ADDED
@@ -0,0 +1,7 @@
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+
5
+ bundle install
6
+
7
+ # Do any other automated setup that you need to do here
data/lib/netgrep/ip.rb ADDED
@@ -0,0 +1,51 @@
1
+ module Netgrep
2
+ # NOTE: the regexes are from Resolv::IPv4 in ruby's stdlib. They were copied here because we
3
+ # need to match IPs in the context of a large string and the Resolv regex has string start
4
+ # and string end anchors which don't allow this.
5
+ module Ipv4
6
+ REGEX_256 = /0
7
+ |1(?:[0-9][0-9]?)?
8
+ |2(?:[0-4][0-9]?|5[0-5]?|[6-9])?
9
+ |[3-9][0-9]?/x
10
+ REGEX = /#{REGEX_256}\.#{REGEX_256}\.#{REGEX_256}\.#{REGEX_256}/x
11
+ end
12
+
13
+ module Ipv6
14
+ # IPv6 address format a:b:c:d:e:f:g:h
15
+ REGEX_8HEX = /
16
+ (?:[0-9A-Fa-f]{1,4}:){7}
17
+ [0-9A-Fa-f]{1,4}
18
+ /x
19
+
20
+ # Compressed IPv6 address format a::b
21
+ REGEX_COMPRESSEDHEX = /
22
+ (?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)? ::
23
+ (?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?
24
+ /x
25
+
26
+ # IPv4 mapped IPv6 address format a:b:c:d:e:f:w.x.y.z
27
+ REGEX_6HEX4DEC = /
28
+ (?:[0-9A-Fa-f]{1,4}:){6,6}
29
+ \d+\.\d+\.\d+\.\d+
30
+ /x
31
+
32
+ # Compressed IPv4 mapped IPv6 address format a::b:w.x.y.z
33
+ REGEX_COMPRESSEDHEX4DEC = /
34
+ (?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)? ::
35
+ (?:[0-9A-Fa-f]{1,4}:)*
36
+ \d+\.\d+\.\d+\.\d+
37
+ /x
38
+
39
+ # A composite IPv6 address Regexp.
40
+ REGEX = /
41
+ (?:#{REGEX_8HEX}) |
42
+ (?:#{REGEX_COMPRESSEDHEX}) |
43
+ (?:#{REGEX_6HEX4DEC}) |
44
+ (?:#{REGEX_COMPRESSEDHEX4DEC})
45
+ /x
46
+ end
47
+
48
+ module Ip
49
+ REGEX = /\b(#{Netgrep::Ipv4::REGEX} | #{Netgrep::Ipv6::REGEX})\b/xi
50
+ end
51
+ end
@@ -0,0 +1,48 @@
1
+ require "ipaddr"
2
+ require "set"
3
+
4
+ module Netgrep
5
+ class Netblocks
6
+
7
+ def initialize(file: nil, netblocks: [])
8
+ @netblocks = file ? File.readlines(file) : netblocks
9
+ end
10
+
11
+ def include?(ip)
12
+ return true if ipv4_addresses.include?(ip)
13
+ ipv6_addresses.any? { |ipv6| ipv6.include?(ip) }
14
+ end
15
+
16
+ private
17
+
18
+ def ipv4_addresses
19
+ @ipv4_addresses ||= begin
20
+ @netblocks.each_with_object(Set.new) do |netblock, ips|
21
+ ip_addr = init_ip(netblock)
22
+ if ip_addr.ipv4?
23
+ ip_addr.to_range.each do |ip|
24
+ ips.add(ip.to_s)
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
30
+
31
+ def ipv6_addresses
32
+ @ipv6_addresses ||= begin
33
+ @netblocks.each_with_object([]) do |netblock, ips|
34
+ ip_addr = init_ip(netblock)
35
+ ips.push(ip_addr) if ip_addr.ipv6?
36
+ end
37
+ end
38
+ end
39
+
40
+ def init_ip(ip)
41
+ begin
42
+ IPAddr.new(ip.strip)
43
+ rescue IPAddr::Error => e
44
+ raise "Invalid Netblock: #{ip.inspect}"
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,29 @@
1
+ module Netgrep
2
+ module Options
3
+
4
+ def self.parse(argv)
5
+ options = OpenStruct.new
6
+ options.netblocks_file = nil
7
+ options.log_file = nil
8
+ options.num_workers = 1
9
+
10
+ parser = OptionParser.new do |opts|
11
+ opts.on("-f NETBLOCKS_FILE", String, "File containing one IP or CIDR per line") do |file_name|
12
+ options.netblocks_file = File.absolute_path(file_name)
13
+ end
14
+
15
+ opts.on("-w NUM_WORKERS", Integer, "Number of worker processes to use") do |n|
16
+ options.num_workers = Integer(n)
17
+ raise(OptionParser::InvalidArgument) if options.num_workers < 1
18
+ end
19
+ end
20
+
21
+ parser.parse!(argv)
22
+ options.log_file = argv[0]
23
+ options.log_file || raise(OptionParser::InvalidOption.new)
24
+
25
+ options
26
+ end
27
+
28
+ end
29
+ end
@@ -0,0 +1,99 @@
1
+ require "time"
2
+ require "ostruct"
3
+
4
+ module Netgrep
5
+ class Runner
6
+ attr_accessor :matches
7
+
8
+ def initialize(log_file:, netblocks_file:, num_workers: 1)
9
+ @log_file = log_file
10
+ @netblocks_file = netblocks_file
11
+ @num_workers = num_workers
12
+ @matches = []
13
+ @worker_pids = []
14
+ @pipe_reader, @pipe_writer = IO.pipe
15
+ end
16
+
17
+ def run
18
+ start_workers
19
+ collect_matches
20
+ shutdown_workers
21
+ print_matches
22
+ self
23
+ end
24
+
25
+ private
26
+
27
+ def start_workers
28
+ num_lines = File.open(@log_file).count
29
+ ranges = calc_line_ranges(num_lines, @num_workers)
30
+
31
+ fd = File.open(@log_file, 'r')
32
+ ranges.each do |range|
33
+ data = select_data_in_range(fd, *range)
34
+ @worker_pids << fork do
35
+ @pipe_reader.close
36
+ start_worker(@pipe_writer, netblocks, data)
37
+ end
38
+ end
39
+
40
+ fd.close
41
+ @pipe_writer.close
42
+ end
43
+
44
+ def calc_line_ranges(num_lines, num_ranges)
45
+ ranges = []
46
+ range_length = num_lines / num_ranges
47
+
48
+ start_line, end_line = 0, range_length-1
49
+ num_ranges.times do
50
+ ranges << [start_line, end_line]
51
+ start_line, end_line = end_line + 1, end_line + range_length
52
+ end
53
+ ranges << [ranges.pop[0], num_lines-1]
54
+ end
55
+
56
+ def select_data_in_range(fd, first_lineno, last_lineno)
57
+ lineno = first_lineno
58
+ data = []
59
+ while lineno <= last_lineno
60
+ data.push(fd.gets)
61
+ lineno += 1
62
+ end
63
+ data
64
+ end
65
+
66
+ def netblocks
67
+ @netblocks ||= Netgrep::Netblocks.new(file: @netblocks_file)
68
+ end
69
+
70
+ def shutdown_workers
71
+ @worker_pids.each { Process.wait }
72
+ rescue Errno::ECHILD
73
+ # noop
74
+ end
75
+
76
+ def print_matches
77
+ @matches.each { |match| $stdout.puts(match) }
78
+ end
79
+
80
+ def collect_matches
81
+ while (match = @pipe_reader.gets)
82
+ @matches.push(match)
83
+ end
84
+ @pipe_reader.close
85
+ end
86
+
87
+ def start_worker(writer, netblocks, data)
88
+ data.each do |line|
89
+ line.scan(Netgrep::Ip::REGEX).each do |md|
90
+ ip = md[0]
91
+ if netblocks.include?(ip)
92
+ writer.write(line)
93
+ break
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,3 @@
1
+ module Netgrep
2
+ VERSION = "0.1.0"
3
+ end
data/lib/netgrep.rb ADDED
@@ -0,0 +1,23 @@
1
+ require "ipaddr"
2
+ require "optparse"
3
+
4
+ require_relative "netgrep/ip"
5
+ require_relative "netgrep/options"
6
+ require_relative "netgrep/netblocks"
7
+ require_relative "netgrep/runner"
8
+ require_relative "netgrep/version"
9
+
10
+
11
+ module Netgrep
12
+ def self.root
13
+ @root ||= File.expand_path(File.join(File.dirname(__FILE__), ".."))
14
+ end
15
+
16
+ def self.fixture_path
17
+ File.expand_path(File.join(root, "spec", "fixtures"))
18
+ end
19
+
20
+ def self.fixture_path_for(file)
21
+ File.expand_path(File.join(fixture_path, file))
22
+ end
23
+ end
data/netgrep.gemspec ADDED
@@ -0,0 +1,32 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'netgrep/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "netgrep"
8
+ spec.version = Netgrep::VERSION
9
+ spec.authors = ["Steven Hansen"]
10
+ spec.email = ["sahglie@gmail.com"]
11
+
12
+ spec.summary = %q{Finds matching lines in a file that contains IPs that fall within specified subnets}
13
+ spec.description = %q{}
14
+ spec.homepage = ""
15
+
16
+ # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
17
+ # delete this section to allow pushing this gem to any host.
18
+ if spec.respond_to?(:metadata)
19
+ spec.metadata['allowed_push_host'] = "https://rubygems.org"
20
+ else
21
+ raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
22
+ end
23
+
24
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
25
+ spec.bindir = "exe"
26
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
27
+ spec.require_paths = ["lib"]
28
+
29
+ spec.add_development_dependency "bundler", "~> 1.10"
30
+ spec.add_development_dependency "rake", "~> 10.0"
31
+ spec.add_development_dependency "rspec"
32
+ end
metadata ADDED
@@ -0,0 +1,104 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: netgrep
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Steven Hansen
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2016-01-15 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.10'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.10'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description: ''
56
+ email:
57
+ - sahglie@gmail.com
58
+ executables: []
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - ".gitignore"
63
+ - ".rspec"
64
+ - ".ruby-version"
65
+ - Gemfile
66
+ - README.md
67
+ - Rakefile
68
+ - bin/console
69
+ - bin/netblocks
70
+ - bin/netgrep
71
+ - bin/setup
72
+ - lib/netgrep.rb
73
+ - lib/netgrep/ip.rb
74
+ - lib/netgrep/netblocks.rb
75
+ - lib/netgrep/options.rb
76
+ - lib/netgrep/runner.rb
77
+ - lib/netgrep/version.rb
78
+ - netgrep.gemspec
79
+ homepage: ''
80
+ licenses: []
81
+ metadata:
82
+ allowed_push_host: https://rubygems.org
83
+ post_install_message:
84
+ rdoc_options: []
85
+ require_paths:
86
+ - lib
87
+ required_ruby_version: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - ">="
90
+ - !ruby/object:Gem::Version
91
+ version: '0'
92
+ required_rubygems_version: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ requirements: []
98
+ rubyforge_project:
99
+ rubygems_version: 2.4.5.1
100
+ signing_key:
101
+ specification_version: 4
102
+ summary: Finds matching lines in a file that contains IPs that fall within specified
103
+ subnets
104
+ test_files: []