hidden-hippo 0.1.0

data/lib/hidden_hippo/dossier.rb

@@ -0,0 +1,23 @@
+require 'mongoid'
+require 'hidden_hippo/possibilities'
+
+module HiddenHippo
+  # Holds all the information for a specific mac address
+  class Dossier
+    include Mongoid::Document
+
+    field :_id, type: String, default: ->{ mac_address }
+
+    field :mac_address, type: String
+    validates_presence_of :mac_address
+
+    field :name, type: Possibilities, default: ->{ Possibilities.new }
+    field :hostname, type: Possibilities, default: ->{ Possibilities.new }
+    field :username, type: Possibilities, default: ->{ Possibilities.new }
+    field :email, type: Possibilities, default: ->{ Possibilities.new }
+    field :device, type: Possibilities, default: ->{ Possibilities.new }
+    field :gender, type: Possibilities, default: ->{ Possibilities.new }
+    field :age, type: Possibilities, default: ->{ Possibilities.new }
+    field :history, type: Possibilities, default: ->{ Possibilities.new }
+  end
+end

data/lib/hidden_hippo/extractors/dhcp_hostname_extractor.rb

@@ -0,0 +1,16 @@
+require 'hidden_hippo/update'
+
+module HiddenHippo
+  module Extractors
+    class DhcpHostnameExtractor
+      def initialize(queue)
+        @queue = queue
+      end
+
+      def call(packet)
+        @queue << Update.new(packet.mac_src,
+                             {hostname: packet.fqdn_name || packet.opt_hostname})
+      end
+    end
+  end
+end

data/lib/hidden_hippo/extractors/dns_history_extractor.rb

@@ -0,0 +1,25 @@
+require 'hidden_hippo/update'
+
+module HiddenHippo
+  module Extractors
+    class DnsHistoryExtractor
+      def initialize(queue)
+        @queue = queue
+      end
+
+      def call(packet)
+        return unless packet.udp_dest_port == 53
+        return if packet.dns_qry_name.delete(" ") == ""
+
+        if packet.request?
+          @queue << Update.new(packet.mac_src, {history: packet.dns_qry_name})
+        end
+
+        if packet.response?
+          @queue << Update.new(packet.mac_dest, {history: packet.dns_qry_name})
+        end
+
+       end
+    end
+  end
+end

data/lib/hidden_hippo/extractors/dns_llmnr_extractor.rb

@@ -0,0 +1,18 @@
+require 'hidden_hippo/update'
+
+module HiddenHippo
+  module Extractors
+    class DnsLlmnrExtractor
+      def initialize(queue)
+        @queue = queue
+      end
+
+      def call(packet)
+        return unless packet.udp_dest_port == 5355
+        return unless packet.request?
+
+        @queue << Update.new(packet.mac_src, {hostname: packet.dns_qry_name})
+      end
+    end
+  end
+end

data/lib/hidden_hippo/extractors/http_request_url_extractor.rb

@@ -0,0 +1,15 @@
+require 'hidden_hippo/update'
+
+module HiddenHippo
+  module Extractors
+    class HttpRequestUrlExtractor
+      def initialize(queue)
+        @queue = queue
+      end
+
+      def call(packet)
+        @queue << Update.new(packet.mac_src, {history: packet.full_uri})
+      end
+    end
+  end
+end

data/lib/hidden_hippo/extractors/mdns_hostname_extractor.rb

@@ -0,0 +1,18 @@
+require 'hidden_hippo/update'
+
+module HiddenHippo
+  module Extractors
+    class MdnsHostnameExtractor
+      def initialize(queue)
+        @queue = queue
+      end
+
+      def call(packet)
+        return unless packet.udp_dest_port == 5353
+        return unless packet.response?
+
+        @queue << Update.new(packet.mac_src, {hostname: packet.ptr})
+      end
+    end
+  end
+end

data/lib/hidden_hippo/gui.rb

@@ -0,0 +1,21 @@
+require 'sinatra/base'
+require 'hidden_hippo'
+require 'hidden_hippo/dossier'
+
+module HiddenHippo
+  class Gui < Sinatra::Application
+    set :root, proc { (HiddenHippo.gem_root + 'gui').to_s }
+
+    get '/' do
+      dossiers = Dossier.all
+
+      erb :index, locals: {dossiers: dossiers}
+    end
+
+    get '/:mac' do
+      dossier = Dossier.find params[:mac]
+
+      erb :dossier, locals: {dossier: dossier}
+    end
+  end
+end

data/lib/hidden_hippo/packets/dhcp.rb

@@ -0,0 +1,13 @@
+require 'hidden_hippo/packets/packet'
+
+module HiddenHippo
+  module Packets
+    class Dhcp < Packet
+      filter 'bootp'
+
+      field :fqdn_name, tshark: 'bootp.fqdn.name'
+      field :opt_hostname, tshark: 'bootp.option.hostname'
+
+    end
+  end
+end

data/lib/hidden_hippo/packets/dns.rb

@@ -0,0 +1,23 @@
+require 'hidden_hippo/packets/packet'
+
+module HiddenHippo
+  module Packets
+    class Dns < Packet
+      filter 'dns'
+
+      field :a, tshark: 'dns.a'
+      field :ptr, tshark: 'dns.ptr.domain_name'
+      field :dns_qry_name, tshark: 'dns.qry.name'
+
+      field :response, tshark: 'dns.flags.response', conv: ->(f) {f.to_i == 1}
+
+      def response?
+        response
+      end
+
+      def request?
+        !response?
+      end
+    end
+  end
+end

data/lib/hidden_hippo/packets/http.rb

@@ -0,0 +1,13 @@
+require 'hidden_hippo/packets/packet'
+
+module HiddenHippo
+  module Packets
+    class Http < Packet
+      filter 'http'
+
+      field :full_uri, tshark: 'http.request.full_uri'
+      field :referer, tshark: 'http.referer'
+
+    end
+  end
+end

data/lib/hidden_hippo/packets/packet.rb

@@ -0,0 +1,73 @@
+module HiddenHippo
+  module Packets
+    Field = Struct.new :name, :tshark_field, :converter
+
+    class Packet
+      def self.fields
+        unless @fields
+          super_fields = self == Packet ? [] : superclass.fields.dup
+          @fields = super_fields
+        end
+        @fields
+      end
+
+      def self.tshark_fields
+        fields.map &:tshark_field
+      end
+
+      def self.filter(filter=nil)
+        @filter = filter if filter
+        @filter
+      end
+
+      def self.field(name, options={})
+        fields << Field.new(name, options[:tshark], options[:conv])
+        attr_accessor name
+        name
+      end
+
+      def self.parse(hash)
+        instance = new
+
+        reverse_names = Hash[*fields.map{|f| [f.tshark_field, f]}.flatten]
+
+        hash.each do |k, v|
+          field = reverse_names[k]
+
+          next unless field
+
+          conv = converter field
+          instance.public_send "#{field.name}=", conv.(v)
+        end
+
+        instance
+      end
+
+      # default fields
+      field :eth_mac_src, tshark: 'eth.src'
+      field :eth_mac_dest, tshark: 'eth.dst'
+      field :wlan_mac_src, tshark: 'wlan.sa'
+      field :wlan_mac_dest, tshark: 'wlan.da'
+
+      field :udp_src_port, tshark: 'udp.srcport', conv: :to_i
+      field :tcp_src_port, tshark: 'tcp.srcport', conv: :to_i
+      field :udp_dest_port, tshark: 'udp.dstport', conv: :to_i
+      field :tcp_dest_port, tshark: 'tcp.dstport', conv: :to_i
+
+      def mac_src
+        eth_mac_src || wlan_mac_src
+      end
+
+      def mac_dest
+        eth_mac_dest || wlan_mac_dest
+      end
+
+      private
+
+      def self.converter(field)
+        conv = field.converter || ->(x){x}
+        conv.is_a?(Symbol) ? conv.to_proc : conv
+      end
+    end
+  end
+end

data/lib/hidden_hippo/paths.rb

@@ -0,0 +1,15 @@
+require 'pathname'
+
+module HiddenHippo
+  module Paths
+    def home
+      @home ||= if ENV['HIDDEN_HIPPO_HOME']
+                  Pathname.new(ENV['HIDDEN_HIPPO_HOME'])
+                else
+                  Pathname.new(ENV['HOME']) + '.hidden-hippo'
+                end
+      @home.mkpath
+      @home
+    end
+  end
+end

data/lib/hidden_hippo/possibilities.rb

@@ -0,0 +1,63 @@
+module HiddenHippo
+  # Represents multiple possible values
+  #
+  # Each possible value has a support count. Adding possible values is done with #<<.
+  # Retrieving the support count for a possible value is done with #[].
+  class Possibilities
+    include Enumerable
+
+    class << self
+      def demongoize(doc)
+        HiddenHippo::Possibilities.new doc
+      end
+
+      def mongoize(value)
+        case value
+          when Possibilities then value.mongoize
+          else value
+        end
+      end
+
+      def evolve(value)
+        case value
+          when Possibilities then value.mongoize
+          else value
+        end
+      end
+
+      def resizable?
+        true
+      end
+    end
+
+    def initialize(counts={})
+      @counts = counts
+      @counts.keep_if {|_, c| c.kind_of?(Integer)}
+      @counts.default_proc = proc {0}
+    end
+
+    def [](value)
+      @counts[value]
+    end
+
+    def <<(value)
+      @counts[value.to_s.gsub(/[.$]/, '_')] += 1
+    end
+
+    def each(&block)
+      @counts.sort_by{|c| - c.last}.map(&:first).each &block
+    end
+
+    def each_with_support(&block)
+      @counts.sort_by{|c| - c.last}.each &block
+    end
+
+    def mongoize
+      @counts
+    end
+
+    def resizable?
+      Possibilities.resizable?
+    end
+  end
+end

data/lib/hidden_hippo/reader.rb

@@ -0,0 +1,36 @@
+require 'hidden_hippo/scanner'
+require 'hidden_hippo/updator'
+
+require 'hidden_hippo/packets/dns'
+require 'hidden_hippo/packets/dhcp'
+require 'hidden_hippo/packets/http'
+
+require 'hidden_hippo/extractors/mdns_hostname_extractor'
+require 'hidden_hippo/extractors/dhcp_hostname_extractor'
+require 'hidden_hippo/extractors/http_request_url_extractor'
+require 'hidden_hippo/extractors/dns_llmnr_extractor'
+require 'thread'
+
+module HiddenHippo
+  class Reader
+    def initialize(file)
+      @file = file
+      updator_queue = Queue.new
+      @updator = Updator.new updator_queue
+      @scanners = []
+      @scanners << Scanner.new(file, Packets::Dns,
+                               Extractors::MdnsHostnameExtractor.new(updator_queue),
+                               Extractors::DnsLlmnrExtractor.new(updator_queue))
+      @scanners << Scanner.new(file, Packets::Dhcp,
+                               Extractors::DhcpHostnameExtractor.new(updator_queue))
+      @scanners << Scanner.new(file, Packets::Http,
+                               Extractors::HttpRequestUrlExtractor.new(updator_queue))
+    end
+
+    def call
+      @updator.start
+      @scanners.each &:call
+      @updator.stop
+    end
+  end
+end

data/lib/hidden_hippo/scanner.rb

@@ -0,0 +1,51 @@
+require 'open3'
+
+module HiddenHippo
+  class Scanner
+    def initialize(file, packet_class, *extractors)
+      @file = file
+      @extractors = extractors
+      @packet_class = packet_class
+    end
+
+    def call
+      # call Tshark
+      tshark_fields = @packet_class.tshark_fields
+
+      args = [
+          '-2', '-Tfields', '-q',
+          '-r', @file,
+          '-R', @packet_class.filter,
+          *tshark_fields.map {|f| ['-e', f]}.flatten
+      ]
+
+      Open3.popen3(%w(tshark tshark), *args) do |stdin, stdout, stderr, waiter|
+        # we don't need those
+        stdin.close
+
+        stdout.each do |line|
+          if line.count("\t") != tshark_fields.size - 1
+            puts 'Warinig: tshark returned a line of the wrong size. Ignoring it.'
+            puts "Offending line: #{line}"
+            next
+          end
+
+          split_line = line.chomp.split("\t").map {|f| f.empty? ? nil : f}
+
+          assoc = tshark_fields.zip split_line
+          packet = @packet_class.parse Hash[*assoc.flatten]
+
+          @extractors.each do |extractor|
+            extractor.call(packet)
+          end
+        end
+
+        if waiter.value != 0
+          puts "Warning: tshark exited with status code #{waiter.value}."
+          puts "tshark #{args.join(' ')}"
+          puts stderr.read
+        end
+      end
+    end
+  end
+end

data/lib/hidden_hippo/update.rb

@@ -0,0 +1,3 @@
+module HiddenHippo
+  Update = Struct.new :mac_address, :fields
+end

data/lib/hidden_hippo/updator.rb

@@ -0,0 +1,49 @@
+require 'thread'
+require 'hidden_hippo/dossier'
+
+module HiddenHippo
+  class Updator
+    def initialize(queue)
+      @queue = queue
+    end
+
+    def start
+      return if @thread
+      @thread = Thread.new do
+        loop do
+          update = @queue.pop
+
+          break if update == :stop
+
+          dossier = dossier update.mac_address
+
+          update.fields.each do |key, value|
+            dossier.public_send(key) << value
+          end
+
+          begin
+          dossier.save
+          rescue => e
+            puts 'shit'
+            p e
+          end
+        end
+      end
+    end
+
+    def stop
+      @queue << :stop
+      @thread.join
+    end
+
+    private
+
+    def dossier(mac_address)
+      if Dossier.where(mac_address: mac_address).exists?
+        Dossier.find mac_address
+      else
+        Dossier.new mac_address: mac_address
+      end
+    end
+  end
+end

data/lib/hidden_hippo/version.rb

@@ -0,0 +1,3 @@
+module HiddenHippo
+  VERSION = '0.1.0'
+end

data/lib/hidden_hippo.rb

@@ -0,0 +1,23 @@
+require 'pathname'
+require 'hidden_hippo/version'
+require 'hidden_hippo/cli/app'
+
+module HiddenHippo
+  def pid_exists?(pid)
+    Process.kill 0, pid
+    return true
+  rescue
+    return false
+  end
+  module_function :pid_exists?
+
+  def configure_db!(env = :production)
+    Mongoid.load! gem_root + 'config/mongoid.yml', env
+  end
+  module_function :configure_db!
+
+  def gem_root
+    Pathname.new(__FILE__) + '../../'
+  end
+  module_function :gem_root
+end

data/spec/db_daemon_spec.rb

@@ -0,0 +1,7 @@
+require 'support/cli_controller_examples'
+
+describe 'hh db', :noisy do
+  let(:name) {'db'}
+
+  it_behaves_like 'cli daemon controller'
+end

data/spec/dns_scanner_spec.rb

@@ -0,0 +1,41 @@
+require 'hidden_hippo/scanner'
+require 'hidden_hippo/packets/dns'
+
+describe 'HiddenHippo::Scanner(Dns)' do
+
+  describe '#call' do
+    it 'should parse the pcap file' do
+      extractor = double('extractor')
+      scanner = HiddenHippo::Scanner.new('spec/fixtures/dns_elise.pcap', HiddenHippo::Packets::Dns, extractor)
+
+      expect(extractor).to receive(:call) do |packet|
+        expect(packet.ptr).to eq 'Elises-MacBook-Pro.local'
+        expect(packet.mac_src).to eq 'a8:bb:cf:09:9b:bc'
+        expect(packet.mac_dest).to eq '01:00:5e:00:00:fb'
+      end
+
+      scanner.call
+    end
+
+    it 'should skip non dns lines' do
+      extractor = double('extractor')
+      scanner = HiddenHippo::Scanner.new 'spec/fixtures/tcp_noise.pcap', HiddenHippo::Packets::Dns, extractor
+
+      expect(extractor).not_to receive(:call)
+
+      scanner.call
+    end
+
+    it 'should find the mac addresses in eth traffic' do
+      extractor = double('extractor')
+      scanner = HiddenHippo::Scanner.new('spec/fixtures/dns_reddit_eth.pcap', HiddenHippo::Packets::Dns, extractor)
+
+      expect(extractor).to receive(:call).at_least(:once) do |packet|
+        expect(packet.mac_src).not_to be_nil
+        expect(packet.mac_dest).not_to be_nil
+      end
+
+      scanner.call
+    end
+  end
+end

data/spec/dossier_spec.rb

@@ -0,0 +1,72 @@
+require 'hidden_hippo/dossier'
+
+describe HiddenHippo::Dossier, :db do
+  before do
+    HiddenHippo::Dossier.delete_all
+  end
+
+  it 'should require a mac address' do
+    expect(HiddenHippo::Dossier.new).to validate_presence_of :mac_address
+  end
+
+  it 'should use the mac address as an id' do
+    dossier = HiddenHippo::Dossier.new mac_address: '00:aa:bb:cc:dd:ee:ff'
+    dossier.save!
+
+    expect(HiddenHippo::Dossier.find('00:aa:bb:cc:dd:ee:ff')).to eq dossier
+  end
+
+  it 'should allow possibilities to be modified in place' do
+    dossier = HiddenHippo::Dossier.new mac_address: 'aa:bb:cc:dd:ee:ff:aa'
+
+    dossier.name << 'Steve'
+
+    expect(dossier.name).to contain_exactly 'Steve'
+  end
+
+  it 'should serialize properly' do
+    dossier = HiddenHippo::Dossier.new(mac_address: '11:00:11:00:11:00:11')
+
+    dossier.name << 'John'
+    dossier.name << 'Billy'
+    dossier.name << 'Billy'
+    dossier.name << 'Steve'
+
+    dossier.username << 'billydakid89'
+    dossier.username << 'billydakid89'
+    dossier.username << 'billydakid89'
+
+    dossier.age << 40
+    dossier.age << 41
+    dossier.age << 41
+    dossier.age << 42
+
+    dossier.save!
+
+    found = HiddenHippo::Dossier.find '11:00:11:00:11:00:11'
+    expect(dossier).to eq found
+    expect(found.name).to contain_exactly 'John', 'Billy', 'Steve'
+    expect(found.username).to contain_exactly 'billydakid89'
+    expect(found.age).to contain_exactly '40', '41', '42'
+  end
+
+  it 'should deserialize support counts as ints' do
+    dossier = HiddenHippo::Dossier.new(mac_address: '11:00:11:00:11:00:11')
+
+    dossier.name << 'Whoopie'
+
+    dossier.save!
+
+    found = HiddenHippo::Dossier.find '11:00:11:00:11:00:11'
+    expect(found.name['Whoopie']).to eq 1
+  end
+
+  it 'should update possibilities' do
+    HiddenHippo::Dossier.create mac_address: 'find me'
+    dossier = HiddenHippo::Dossier.find 'find me'
+    dossier.name << 'thing'
+    dossier.save
+
+    expect(HiddenHippo::Dossier.find('find me').name['thing']).to eq 1
+  end
+end

data/spec/extractors/dhcp_hostname_extractor_spec.rb

@@ -0,0 +1,43 @@
+require 'thread'
+require 'hidden_hippo/extractors/dhcp_hostname_extractor'
+
+describe HiddenHippo::Extractors::DhcpHostnameExtractor do
+  let(:queue) {Queue.new}
+  let(:extractor) {HiddenHippo::Extractors::DhcpHostnameExtractor.new queue}
+
+  describe '#call' do
+    it 'should not have a fqdn_name' do
+      packet = HiddenHippo::Packets::Dhcp.new
+      packet.eth_mac_src = 'dhcp some mac'
+      packet.fqdn_name = nil
+      packet.opt_hostname = 'hidden_hippo_super_mac'
+
+      extractor.call packet
+      out = queue.pop true
+      expect(out.mac_address).to eq 'dhcp some mac'
+      expect(out.fields).to eq ({hostname: 'hidden_hippo_super_mac'})
+    end
+    it 'should use a fqdn_name' do
+      packet = HiddenHippo::Packets::Dhcp.new
+      packet.eth_mac_src = 'dhcp some mac'
+      packet.fqdn_name = 'hidden_hippo_fqdn'
+      packet.opt_hostname = 'hidden_hippo_super_mac'
+
+      extractor.call packet
+      out = queue.pop true
+      expect(out.mac_address).to eq 'dhcp some mac'
+      expect(out.fields).to eq ({hostname: 'hidden_hippo_fqdn'})
+    end
+    it 'should not have a hostname' do
+      packet = HiddenHippo::Packets::Dhcp.new
+      packet.eth_mac_src = 'dhcp some mac'
+      packet.fqdn_name = 'hidden_hippo_super_mac'
+      packet.opt_hostname = nil
+
+      extractor.call packet
+      out = queue.pop true
+      expect(out.mac_address).to eq 'dhcp some mac'
+      expect(out.fields).to eq ({hostname: 'hidden_hippo_super_mac'})
+    end
+  end
+end