netgrep 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0379650c8ebcdb1a4e2e72db58fe1b1ada7e3886
+  data.tar.gz: b6e24cf116aede9c004872be2b9938a9d95530fc
+SHA512:
+  metadata.gz: a9ef509196130027e388fd334cf41a032a8319c36d8e75ce4cb4ba25b7f66033cb5d49c71e69d5d8c855f09b6695b995054b8f1d0a34ab36157664600f06ba57
+  data.tar.gz: 5e081fb5a201420391c8ce2e84a77a16f589e8d4ceb4306c99b9d520b1f4eb08cf1dcb2a4de12167a25e474e996f96e2f9788bff2c395608435739337deafd0d

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/spec/fixtures/fsb-large.log
+/.idea

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-version

@@ -0,0 +1 @@
+2.2.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in netgrep.gemspec
+gemspec

data/README.md

@@ -0,0 +1,36 @@
+# Netgrep
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/netgrep`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'netgrep'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install netgrep
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/netgrep.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "netgrep"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/netblocks

@@ -0,0 +1,43 @@
+2607:f140:1:8003:6a5b:35ff:fe85:90f4/64
+10.254.1.0/24
+10.254.10.0/24
+10.254.13.0/24
+10.254.15.0/24
+10.254.16.0/24
+10.254.18.0/23
+10.254.2.0/24
+10.254.20.0/24
+10.254.25.0/24
+10.254.26.0/24
+10.254.27.0/24
+10.254.28.0/24
+10.254.3.0/24
+10.254.4.0/24
+10.254.9.0/24
+10.35.1.0
+128.32.16.64/26
+128.32.169.128/25
+128.32.185.0/26
+128.32.189.0/24
+128.32.209.0/24
+128.32.249.0/24
+128.32.25.0/24
+128.32.31.0/24
+128.32.80.32/27
+169.229.131.0/25
+169.229.161.0/25
+169.229.164.0/24
+169.229.217.160/27
+169.229.217.192/27
+169.229.218.0/26
+169.229.219.0/26
+169.229.243.0/26
+169.229.56.128/26
+132.249.243.0/24
+192.31.161.0/24
+192.31.95.128/27
+192.31.95.160/27
+192.31.95.192/26
+192.31.95.96/27
+192.58.221.0/25
+192.58.221.128/25

data/bin/netgrep

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+# Exit cleanly from an early interrupt
+Signal.trap("INT") { exit 1 }
+
+require "bundler/setup"
+require "netgrep"
+
+begin
+  opts = Netgrep::Options.parse(ARGV)
+  runner = Netgrep::Runner.new(
+      log_file: opts.log_file,
+      netblocks_file: opts.netblocks_file,
+      num_workers: opts.num_workers
+  )
+  runner.run
+rescue Errno::ENOENT => err
+  abort "netgrep: #{err.message}"
+rescue OptionParser::InvalidOption, OptionParser::InvalidArgument
+  abort "usage: netgrep [-w NUM_WORKERS] -f [NETBLOCKS_FILE] [LOG_FILE]"
+end

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/netgrep/ip.rb

@@ -0,0 +1,51 @@
+module Netgrep
+  # NOTE: the regexes are from Resolv::IPv4 in ruby's stdlib.  They were copied here because we
+  # need to match IPs in the context of a large string and the Resolv regex has string start
+  # and string end anchors which don't allow this.
+  module Ipv4
+    REGEX_256 = /0
+               |1(?:[0-9][0-9]?)?
+               |2(?:[0-4][0-9]?|5[0-5]?|[6-9])?
+               |[3-9][0-9]?/x
+    REGEX = /#{REGEX_256}\.#{REGEX_256}\.#{REGEX_256}\.#{REGEX_256}/x
+  end
+
+  module Ipv6
+    # IPv6 address format a:b:c:d:e:f:g:h
+    REGEX_8HEX = /
+      (?:[0-9A-Fa-f]{1,4}:){7}
+         [0-9A-Fa-f]{1,4}
+    /x
+
+    # Compressed IPv6 address format a::b
+    REGEX_COMPRESSEDHEX = /
+      (?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)? ::
+      (?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?
+    /x
+
+    # IPv4 mapped IPv6 address format a:b:c:d:e:f:w.x.y.z
+    REGEX_6HEX4DEC = /
+      (?:[0-9A-Fa-f]{1,4}:){6,6}
+      \d+\.\d+\.\d+\.\d+
+    /x
+
+    # Compressed IPv4 mapped IPv6 address format a::b:w.x.y.z
+    REGEX_COMPRESSEDHEX4DEC = /
+      (?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)? ::
+      (?:[0-9A-Fa-f]{1,4}:)*
+      \d+\.\d+\.\d+\.\d+
+    /x
+
+    # A composite IPv6 address Regexp.
+    REGEX = /
+      (?:#{REGEX_8HEX}) |
+      (?:#{REGEX_COMPRESSEDHEX}) |
+      (?:#{REGEX_6HEX4DEC}) |
+      (?:#{REGEX_COMPRESSEDHEX4DEC})
+    /x
+  end
+
+  module Ip
+    REGEX = /\b(#{Netgrep::Ipv4::REGEX} | #{Netgrep::Ipv6::REGEX})\b/xi
+  end
+end

data/lib/netgrep/netblocks.rb

@@ -0,0 +1,48 @@
+require "ipaddr"
+require "set"
+
+module Netgrep
+  class Netblocks
+
+    def initialize(file: nil, netblocks: [])
+      @netblocks = file ? File.readlines(file) : netblocks
+    end
+
+    def include?(ip)
+      return true if ipv4_addresses.include?(ip)
+      ipv6_addresses.any? { |ipv6| ipv6.include?(ip) }
+    end
+
+    private
+
+    def ipv4_addresses
+      @ipv4_addresses ||= begin
+        @netblocks.each_with_object(Set.new) do |netblock, ips|
+          ip_addr = init_ip(netblock)
+          if ip_addr.ipv4?
+            ip_addr.to_range.each do |ip|
+              ips.add(ip.to_s)
+            end
+          end
+        end
+      end
+    end
+
+    def ipv6_addresses
+      @ipv6_addresses ||= begin
+        @netblocks.each_with_object([]) do |netblock, ips|
+          ip_addr = init_ip(netblock)
+          ips.push(ip_addr) if ip_addr.ipv6?
+        end
+      end
+    end
+
+    def init_ip(ip)
+      begin
+        IPAddr.new(ip.strip)
+      rescue IPAddr::Error => e
+        raise "Invalid Netblock: #{ip.inspect}"
+      end
+    end
+  end
+end

data/lib/netgrep/options.rb

@@ -0,0 +1,29 @@
+module Netgrep
+  module Options
+
+    def self.parse(argv)
+      options = OpenStruct.new
+      options.netblocks_file = nil
+      options.log_file = nil
+      options.num_workers = 1
+
+      parser = OptionParser.new do |opts|
+        opts.on("-f NETBLOCKS_FILE", String, "File containing one IP or CIDR per line") do |file_name|
+          options.netblocks_file = File.absolute_path(file_name)
+        end
+
+        opts.on("-w NUM_WORKERS", Integer, "Number of worker processes to use") do |n|
+          options.num_workers = Integer(n)
+          raise(OptionParser::InvalidArgument) if options.num_workers < 1
+        end
+      end
+
+      parser.parse!(argv)
+      options.log_file = argv[0]
+      options.log_file || raise(OptionParser::InvalidOption.new)
+
+      options
+    end
+
+  end
+end

data/lib/netgrep/runner.rb

@@ -0,0 +1,99 @@
+require "time"
+require "ostruct"
+
+module Netgrep
+  class Runner
+    attr_accessor :matches
+
+    def initialize(log_file:, netblocks_file:, num_workers: 1)
+      @log_file = log_file
+      @netblocks_file = netblocks_file
+      @num_workers = num_workers
+      @matches = []
+      @worker_pids = []
+      @pipe_reader, @pipe_writer = IO.pipe
+    end
+
+    def run
+      start_workers
+      collect_matches
+      shutdown_workers
+      print_matches
+      self
+    end
+
+    private
+
+    def start_workers
+      num_lines = File.open(@log_file).count
+      ranges = calc_line_ranges(num_lines, @num_workers)
+
+      fd = File.open(@log_file, 'r')
+      ranges.each do |range|
+        data = select_data_in_range(fd, *range)
+        @worker_pids << fork do
+          @pipe_reader.close
+          start_worker(@pipe_writer, netblocks, data)
+        end
+      end
+
+      fd.close
+      @pipe_writer.close
+    end
+
+    def calc_line_ranges(num_lines, num_ranges)
+      ranges = []
+      range_length = num_lines / num_ranges
+
+      start_line, end_line = 0, range_length-1
+      num_ranges.times do
+        ranges << [start_line, end_line]
+        start_line, end_line = end_line + 1, end_line + range_length
+      end
+      ranges << [ranges.pop[0], num_lines-1]
+    end
+
+    def select_data_in_range(fd, first_lineno, last_lineno)
+      lineno = first_lineno
+      data = []
+      while lineno <= last_lineno
+        data.push(fd.gets)
+        lineno += 1
+      end
+      data
+    end
+
+    def netblocks
+      @netblocks ||= Netgrep::Netblocks.new(file: @netblocks_file)
+    end
+
+    def shutdown_workers
+      @worker_pids.each { Process.wait }
+    rescue Errno::ECHILD
+      # noop
+    end
+
+    def print_matches
+      @matches.each { |match| $stdout.puts(match) }
+    end
+
+    def collect_matches
+      while (match = @pipe_reader.gets)
+        @matches.push(match)
+      end
+      @pipe_reader.close
+    end
+
+    def start_worker(writer, netblocks, data)
+      data.each do |line|
+        line.scan(Netgrep::Ip::REGEX).each do |md|
+          ip = md[0]
+          if netblocks.include?(ip)
+            writer.write(line)
+            break
+          end
+        end
+      end
+    end
+  end
+end

data/lib/netgrep/version.rb

@@ -0,0 +1,3 @@
+module Netgrep
+  VERSION = "0.1.0"
+end

data/lib/netgrep.rb

@@ -0,0 +1,23 @@
+require "ipaddr"
+require "optparse"
+
+require_relative "netgrep/ip"
+require_relative "netgrep/options"
+require_relative "netgrep/netblocks"
+require_relative "netgrep/runner"
+require_relative "netgrep/version"
+
+
+module Netgrep
+  def self.root
+    @root ||= File.expand_path(File.join(File.dirname(__FILE__), ".."))
+  end
+
+  def self.fixture_path
+    File.expand_path(File.join(root, "spec", "fixtures"))
+  end
+
+  def self.fixture_path_for(file)
+    File.expand_path(File.join(fixture_path, file))
+  end
+end

data/netgrep.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'netgrep/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "netgrep"
+  spec.version       = Netgrep::VERSION
+  spec.authors       = ["Steven Hansen"]
+  spec.email         = ["sahglie@gmail.com"]
+
+  spec.summary       = %q{Finds matching lines in a file that contains IPs that fall within specified subnets}
+  spec.description   = %q{}
+  spec.homepage      = ""
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: netgrep
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Steven Hansen
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-01-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- sahglie@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/netblocks
+- bin/netgrep
+- bin/setup
+- lib/netgrep.rb
+- lib/netgrep/ip.rb
+- lib/netgrep/netblocks.rb
+- lib/netgrep/options.rb
+- lib/netgrep/runner.rb
+- lib/netgrep/version.rb
+- netgrep.gemspec
+homepage: ''
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Finds matching lines in a file that contains IPs that fall within specified
+  subnets
+test_files: []