nwsdk 1.1.3

data/README.md

@@ -0,0 +1,64 @@
+# Nwsdk
+
+Simplified wrapper + cli for NetWitness REST endpoints
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'nwsdk'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install nwsdk
+
+## Usage
+Module documentation is non-existent. Best bet is to look at the specs and/or the cli
+driver invocations.
+
+To get up and running, invoke 'nw config' and edit ~/.nwsdk.json
+
+The cli is mainly used from the nw command:
+
+    Commands:
+      nw cef CONDITIONS --loghost=LOGHOST            # send cef alerts for query conditions
+      nw configure [$HOME/.nwsdk.json]               # write out a template configuration file
+      nw content CONDITIONS                          # extract files for given query conditions
+      nw help [COMMAND]                              # Describe available commands or one specific command
+      nw pcap CONDITIONS                             # extract PCAP for given query conditions
+      nw query CONDITIONS                            # execute SDK query
+      nw timeline CONDITIONS                         # get a time-indexed histogram for conditions
+      nw values CONDITIONS                           # get value report for specific meta key
+
+    Options:
+      [--config=CONFIG]  # JSON file with endpoint info & credentials
+                         # Default: $HOME/.nwsdk.json
+      [--host=HOST]      # hostname for broker or concentrator
+      [--port=N]         # REST port for broker/concentrator
+                         # Default: 50103
+      [--span=N]         # max timespan in seconds
+                         # Default: 3600
+      [--limit=N]        # max number of sessions
+                         # Default: 10000
+      [--start=START]    # start time for query
+                         # Default: $now-1h
+      [--end=END]        # end time for query
+                         # Default: $now-ish
+
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ryanbreed/nwsdk.
+
+Any fixtures/mocks/etc for the actual REST traffic would be highly welcome additions.
+
+## License
+
+GPLv3 (see LICENSE)

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/nw

@@ -0,0 +1,7 @@
+#!/bin/env ruby
+$LOAD_PATH.push(File.join(File.dirname(__FILE__), "..", "lib"))
+require 'nwsdk'
+require 'nwsdk/cli'
+
+
+Nwsdk::Cli.start(ARGV)

data/lib/nwsdk/cli.rb

@@ -0,0 +1,210 @@
+require 'thor'
+require 'chronic'
+require 'nwsdk'
+require 'cef'
+require 'fileutils'
+
+module Nwsdk
+  def self.setup_cli(options, where)
+    endpoint=Nwsdk::Endpoint.configure(options[:config])
+    e=case options[:end]
+      when nil?
+        options[:start] + options[:span]
+      else
+        Chronic.parse(options[:end])
+    end
+
+
+
+    condition=Nwsdk::Condition.new(
+      where: where,
+      start: Chronic.parse(options[:start]),
+      end:   e,
+      span:  options[:span]
+    )
+    { endpoint: endpoint, condition: condition }
+  end
+
+  class Cli < Thor
+    now=Time.new
+    class_option :config,
+      default: File.join(ENV['HOME'],'.nwsdk.json'),
+      desc:    'JSON file with endpoint info & credentials'
+    class_option :host,
+      desc:      'hostname for broker or concentrator',
+      default: ENV['NW_HOST']
+    class_option :port,
+      type: :numeric,
+      desc:    'REST port for broker/concentrator',
+      default: 50103
+    # TODO: fix this
+    class_option :span,
+      type: :numeric,
+      default: 3600,
+      desc: 'max timespan in seconds'
+    class_option :limit,
+      type: :numeric,
+      default: 10000,
+      desc: 'max number of sessions'
+    # TODO: fix this
+    class_option :start,
+      desc: 'start time for query',
+      type: :string,
+      default: (now - 3600).strftime('%Y-%m-%d %H:%M:%S')
+
+    class_option :end,
+      desc: 'end time for query',
+      type: :string
+    class_option :debug,
+      type: :boolean,
+      default: false,
+      desc: 'extra info'
+
+
+    desc 'timeline', 'get a time-indexed histogram of sessions/packets/bytes'
+    option :where,
+      desc: 'search condition, e.g. ip.src=1.1.1.1&&service=0',
+      type: :string
+    option :flags,
+      type: :string,
+      default: 'size',
+      desc: 'comma-separated list of sessions, size, packets, order-ascending, order-descending'
+    def timeline
+      flags=options[:flags].split(',')
+      timeline=Nwsdk::Timeline.new(Nwsdk.setup_cli(options,options[:where]).merge(flags: flags))
+      result=timeline.request
+      puts JSON.pretty_generate(result)
+    end
+
+    desc 'values CONDITIONS', 'get value report for specific meta key'
+    option :size,
+      type: :numeric,
+      default: 100,
+      desc: 'limit to this number of results'
+    option :key_name,
+      type: :string,
+      default: 'service',
+      desc: 'meta key name'
+    option :where,
+      desc: 'search condition, e.g. ip.src=1.1.1.1&&service=0',
+      type: :string
+    option :flags,
+      default: 'sort-total,sessions,order-descending',
+      type: :string,
+      desc: 'comma-separated combindation of sessions, size, packets, sort-total, sort-value, order-ascending, order-descending'
+    def values
+      flags=options[:flags].split(',')
+      vals=Nwsdk::Values.new(Nwsdk.setup_cli(options,where=options[:where]))
+      vals.key_name=options[:key_name]
+      vals.limit=options[:size]
+      vals.flags=flags
+      result=vals.request
+      puts JSON.pretty_generate(result)
+    end
+
+    desc 'query CONDITIONS', 'execute SDK query'
+    option :keys, default: '*', desc: 'comma-separated list of meta keys to include'
+    def query(where)
+      nwq=Nwsdk::Query.new(Nwsdk.setup_cli(options,where))
+      nwq.keys=options[:keys].split(',')
+      result=nwq.request
+      puts JSON.pretty_generate(result)
+    end
+
+    desc 'content CONDITIONS', 'extract files for given query conditions'
+    option :dir,   default: 'tmp', desc: 'output directory'
+    option :exclude, default: '', desc: 'comma-separated list of file extensions to exclude'
+    option :include, default: '', desc: 'include only this comma-separated list of extensions'
+    def content(where)
+      content=Nwsdk::Content.new(Nwsdk.setup_cli(options,where))
+      content.output_dir=options[:dir]
+      incl=options[:include].split(',')
+      excl=options[:exclude].split(',')
+      content.include_types=incl unless incl==[]
+      content.exclude_types=excl unless excl==[]
+      content.each_session_file do |file|
+        FileUtils.mkdir_p(options[:dir]) unless Dir.exist?(options[:dir])
+        outf=File.join(options[:dir],file[:filename])
+        STDERR.puts "writing #{outf}"
+        File.open(outf,'w') {|f| f.write(file[:data]) }
+      end
+    end
+
+    desc 'pcap CONDITIONS', 'extract PCAP for given query conditions'
+    option :prefix, default: 'pcap_%08d' % $$, desc: 'file name prefix'
+    option :group, type: :numeric, desc: 'max sessions per pcap file', default: 1000
+    def pcap(where)
+      p=Nwsdk::Packets.new(Nwsdk.setup_cli(options,where))
+      p.group=options[:group]
+      p.file_prefix=options[:prefix]
+      p.each_pcap_group do |g|
+        STDERR.puts "Writing #{g[:filename]}"
+        File.open(g[:filename],'w') {|f| f.write(g[:data])}
+      end
+    end
+
+    desc 'cef CONDITIONS', 'send cef alerts for query conditions'
+    option :name,
+      desc:    'name of event',
+      default: 'cef alert'
+    option :loghost,
+      required: true,
+      desc: 'syslog destination'
+    option :logport,
+      default: 514,
+      type:    :numeric,
+      desc: 'syslog UDP port'
+    def cef(where)
+      nwq = Nwsdk::Query.new(Nwsdk.setup_cli(options,where))
+      nwq.keys = ['*']
+      result  = nwq.request
+
+      mapping = nwq.endpoint.config['cef_mapping']
+
+      sender = case nwq.endpoint.loghost
+        when nil
+          CEF::UDPSender.new(options[:loghost],options[:logport])
+        else
+          CEF::UDPSender.new(*nwq.endpoint.loghost)
+      end
+
+      result.each do |res|
+        event=CEF::Event.new
+        event_fields=mapping.keys & res.keys
+        event_fields.each do |field|
+          event.send('%s=' % mapping[field],res[field].to_s)
+        end
+        nwq.endpoint.config['cef_static_fields'].each {|k,v| event.send('%s='%k,v)}
+        event.name=options[:name]
+        event.endTime=(res['time'].to_i * 1000).to_s
+        puts event.to_s
+        sender.emit(event)
+      end
+    end
+
+    config=File.join(ENV['HOME'],'.nwsdk.json')
+    desc 'configure /path/to/config.json', 'write out a template configuration file'
+    option :user,
+      default: 'admin',
+      desc: 'username with sdk query rights'
+    option :pass,
+      default: 'netwitness',
+      desc: 'password'
+    option :loghost,
+      desc: 'syslog destination for cef alerts'
+    option :logport,
+      default: 514,
+      type:    :numeric,
+      desc: 'syslog UDP port'
+    def configure(path=File.join(ENV['HOME'],'.nwsdk.json'))
+      conf=Nwsdk::Constants::DEFAULT_CONFIG.dup
+      conf['endpoint']['host']=options[:host] unless options[:host].nil?
+      conf['endpoint']['port']=options[:port]
+      conf['endpoint']['user']=options[:user]
+      conf['endpoint']['pass']=options[:pass]
+      conf['syslog']['loghost']=options[:loghost] unless options[:loghost].nil?
+      conf['syslog']['logport']=options[:logport]
+      File.open(path,'w') {|f| f.write JSON.pretty_generate(conf) }
+    end
+  end
+end

data/lib/nwsdk/condition.rb

@@ -0,0 +1,76 @@
+module Nwsdk
+  class Condition
+    include Helpers
+    attr_accessor :where, :start, :end, :span, :id1, :id2, :tz_offset
+
+    def initialize(*args)
+      Hash[*args].each {|k,v| self.send("%s="%k, v)}
+      init = Time.new
+
+      # TODO: this is hella ugly, mane
+      if @end.nil?
+        @span ||= 3600
+        if @start.nil?
+          @end=init
+          @start=@end - @span
+        else
+          @end = @start + @span
+        end
+      else
+        if @start.nil?
+          @span ||= 3600
+          @start = @end - @span
+        else
+          @span = @end - @start
+        end
+
+      end
+      @tz_offset = @start.gmtoff
+    end
+
+    def format(use_time: true)
+      formatted_where=case where
+        when nil
+          ""
+        else
+          sprintf('(%s)',where)
+      end
+      if use_time
+        if formatted_where==""
+          sprintf('time=%s', format_time_range)
+        else
+          sprintf('%s&&time=%s',formatted_where,format_time_range)
+        end
+      else
+        formatted_where
+      end
+    end
+
+    def format_time_range
+      sprintf("'%s'-'%s'",
+        format_timestamp(@start),
+        format_timestamp(@end)
+      )
+    end
+
+    # alias for #start
+
+    def time1=(t)
+      @start=t
+    end
+
+    def time1
+      @start
+    end
+
+    # alias for #end
+
+    def time2=(t)
+      @end=t
+    end
+
+    def time2
+      @end
+    end
+  end
+end

data/lib/nwsdk/constants.rb

@@ -0,0 +1,129 @@
+module Nwsdk
+  module Constants
+    NW_TIME_FORMAT = '%Y-%b-%d %H:%M:%S'
+    NW_VARIANT_DAYS = %w{
+      Sunday
+      Monday
+      Tuesday
+      Wednesday
+      Thursday
+      Friday
+      Saturday
+    }
+    #######
+    #
+    # Nwsdk Content Render Types
+    #
+    #######
+
+    NW_CONTENT_TYPE_AUTO           = 0,   # Auto Select HTML Content View
+    NW_CONTENT_TYPE_DETAILS        = 1,   # HTML Meta Details View
+    NW_CONTENT_TYPE_TEXT           = 2,   # HTML Text View
+    NW_CONTENT_TYPE_HEX            = 3,   # HTML Hex View
+    NW_CONTENT_TYPE_PACKETS        = 4,   # HTML Packet View
+    NW_CONTENT_TYPE_MAIL           = 5,   # HTML Mail View
+    NW_CONTENT_TYPE_WEB            = 6,   # HTML Web Page View
+    NW_CONTENT_TYPE_VOIP           = 7,   # HTML VOIP View
+    NW_CONTENT_TYPE_IM             = 8,   # HTML IM View
+    NW_CONTENT_TYPE_FILES          = 9,   # HTML Listing of all files found in session
+    NW_CONTENT_TYPE_PCAP           = 100, # Pcap Packet File
+    NW_CONTENT_TYPE_RAW            = 102, # Raw Content
+    NW_CONTENT_TYPE_XML            = 103, # Meta XML File
+    NW_CONTENT_TYPE_CSV            = 104, # Meta Comma Separated File
+    NW_CONTENT_TYPE_TXT            = 105, # Meta Tab Separated File
+    NW_CONTENT_TYPE_NWD            = 106, # Netwitness Data File
+    NW_CONTENT_TYPE_FILE_EXTRACTOR = 107, # Extract files from common protocols
+    NW_CONTENT_TYPE_LOGS           = 108, # Log extract (captured logs, LF delimited)
+    NW_CONTENT_TYPE_PROTOBUF       = 109  # Content of the NWD as a Google Protocol Buffer object
+
+    ######
+    #
+    #  Nwsdk Content Render Flags
+    #
+    ######
+
+    NW_CONTENT_FLAG_STREAM1          = 0x00001, # Return only request stream
+                                                #   content.
+    NW_CONTENT_FLAG_STREAM2          = 0x00002, # Return only response stream
+                                                #   content.
+    NW_CONTENT_FLAG_SINGLE_COLUMN    = 0x00004, # Format generated web page as
+                                                #   a single column with requests
+                                                #   and responses interleaved.
+    NW_CONTENT_FLAG_PACKET_PAYLOAD   = 0x00008, # Include only session payload.
+    NW_CONTENT_FLAG_DECODEAS_EBCDIC  = 0x00010, # Convert session payload from EBCDIC to ASCII.
+    NW_CONTENT_FLAG_DO_NOT_EMBED     = 0x00020, # Do not embed application/
+                                                #   audio/video traffic into the
+                                                #   generated web page.
+    NW_CONTENT_FLAG_UNCOMPRESS_TEXT  = 0x00040, # Unzip web content in text view.
+    NW_CONTENT_FLAG_DECODE_SSL       = 0x00080, # Attempt to decrypt SSL
+                                                #   sessions if the encryption
+                                                #   key is provided.
+    NW_CONTENT_FLAG_STRIP_STYLE_TAGS = 0x00100, # Removes all <style> tags from the original
+                                                #   html document.
+    NW_CONTENT_FLAG_IGNORE_CACHE     = 0x01000, # Ignore any content in cache
+                                                #   and requery, affects only
+                                                #   current request.
+    NW_CONTENT_FLAG_NO_EMBEDDED_EXE  = 0x02000, # Do not look for or extract
+                                                #   hidden/embedded PE files
+                                                #   when performing file
+                                                #   extraction.
+    NW_CONTENT_FLAG_INCLUDE_DUPS     = 0x04000, # Include packets otherwise removed by assembly
+    NW_CONTENT_FLAG_INCLUDE_HEADERS  = 0x08000, # Include packet header meta information
+    NW_CONTENT_FLAG_CAPTURE_ORDER    = 0x10000  # Do not assemble packets, return in capture order
+
+    ######
+    #
+    # d.'e'fault config
+    #
+    #####
+    DEFAULT_CONFIG = {
+      "endpoint"=> {
+        "user"=>"admin",
+        "pass"=>"netwitness",
+        "host"=>"broker.local",
+        "port"=>"50103"
+      },
+      "syslog"=>{
+        "loghost"=>"loghost.local",
+        "logport"=>514
+      },
+      "cef_static_fields"=> {
+        "deviceVendor"=>"ERCOT",
+        "deviceProduct"=>"nwsdk",
+        "deviceCustomString1Label"=>"threat.desc",
+        "deviceCustomString2Label"=>"threat.source",
+        "deviceCustomString3Label"=>"threat.category",
+        "deviceCustomNumber1Label"=>"asn.src",
+        "deviceCustomNumber2Label"=>"asn.dst"
+      },
+      "cef_mapping" => {
+        "action"=>"deviceAction",
+        "alias.host"=>"destinationProcessName",
+        "asn.dst"=>"deviceCustomNumber2",
+        "asn.src"=>"deviceCustomNumber1",
+        "did"=>"deviceHostName",
+        "directory"=>"filePath",
+        "domain.dst"=>"destinationHostName",
+        "eth.dst"=>"destinationMacAddress",
+        "eth.src"=>"sourceMacAddress",
+        "filename"=>"fileName",
+        "ip.dst"=>"destinationAddress",
+        "ip.proto"=>"transportProtocol",
+        "ip.src"=>"sourceAddress",
+        "risk.warning"=>"name",
+        "risk.suspicious"=>"name",
+        "service"=>"applicationProtocol",
+        "sessionid"=>"externalId",
+        "size"=>"bytesIn",
+        "tcp.dstport"=>"destinationPort",
+        "tcp.srcport"=>"sourcePort",
+        "threat.desc"=>"deviceCustomString1",
+        "threat.source"=>"deviceCustomString2",
+        "threat.category"=>"deviceCustomString3",
+        "udp.dstport"=>"destinationPort",
+        "udp.srcport"=>"sourcePort",
+        "username"=>"destinationUserName"
+      }
+    }
+  end
+end

data/lib/nwsdk/content.rb

@@ -0,0 +1,63 @@
+module Nwsdk
+  class Content
+    include Helpers
+    include Constants
+
+    attr_accessor :output_dir, :include_types, :exclude_types,
+      :condition, :endpoint
+
+    def initialize(*args)
+      Hash[*args].each {|k,v| self.send('%s='%k, v)}
+      @include_types ||= []
+      @exclude_types ||= []
+      @output_dir    ||= 'tmp'
+    end
+
+    def build_request(session)
+      endpoint.get_request(
+        path: 'sdk/content',
+        params: build_params(session)
+      )
+    end
+
+    def each_session
+      sessions=get_sessionids(self)
+      sessions.each do |sessionid|
+        begin
+          response=build_request(sessionid).execute
+        rescue RestClient::Gone
+          next
+        end
+        next unless response.code==200
+        yield response
+      end
+    end
+
+    def each_session_file
+      each_session do |response|
+        content=response.body.encode('BINARY')
+        next unless response.headers.has_key? :content_type
+        boundary=get_boundary(response.headers[:content_type])
+        each_multipart_response_entity(content,boundary) do |file|
+          yield file
+        end
+      end
+    end
+
+    def build_params(session)
+      send_params={
+        session: session,
+        maxSize: 0,
+        render:  NW_CONTENT_TYPE_FILE_EXTRACTOR
+      }
+      unless include_types.empty?
+        send_params[:includeFileTypes]=include_types.join(';')
+      end
+      unless exclude_types.empty?
+        send_params[:excludeFileTypes]=exclude_types.join(';')
+      end
+      send_params
+    end
+
+  end
+end

data/lib/nwsdk/endpoint.rb

@@ -0,0 +1,60 @@
+module Nwsdk
+  class Endpoint
+    attr_accessor :host, :port, :user, :pass, :config, :save_request
+    def self.configure(conf=File.join(ENV["HOME"],".nwsdk.json"))
+      config=JSON.parse(File.read(conf))
+      endpoint=Nwsdk::Endpoint.new(config["endpoint"])
+      endpoint.config=config
+      endpoint
+    end
+
+    def initialize(*args)
+      Hash[*args].each {|k,v| self.send("%s=" % k, v) }
+      @port ||= 50103
+      @save_request||=false
+      yield self if block_given?
+      self
+    end
+
+    def type=(type_sym)
+      @port=case type_sym
+        when :broker
+          50103
+        when :decoder
+          50104
+        when :concentrator
+          50105
+        else
+          raise ArgumentError, "invalid endpoint type #{type_sym.to_s}"
+      end
+    end
+
+    def uri(path='sdk')
+      sprintf('https://%s:%d/%s', host, port, path)
+    end
+
+    def get_request(path: 'sdk', params: {})
+      req=RestClient::Request.new(
+        method:     :get,
+        url:        uri(path),
+        user:       user,
+        password:   pass,
+        read_timenout: nil,
+        verify_ssl: OpenSSL::SSL::VERIFY_NONE,
+        payload: params,
+        headers:  {
+          "Accept-Encoding" => :deflate,
+          accept: :json,
+        }
+      )
+    end
+
+    def loghost
+      if config.has_key?("syslog")
+        [ config["syslog"]["loghost"], config["syslog"]["logport"]]
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/nwsdk/helpers.rb

@@ -0,0 +1,70 @@
+module Nwsdk
+  module Helpers
+    MULTIPART_BOUNDARY=%r{\Amultipart/mixed; boundary="(?<msg_boundary>.+)"\z}
+    MULTIPART_PROLOGUE=%r{\AThis is a message with multiple parts in MIME format.}
+    MULTIPART_END=%r{\A--\z}
+    ATTACHMENT_FILENAME=%r{\Aattachment; filename="(?<filename>.+)"\z}
+
+    def decode_value(field)
+      case field['format']
+      when 1,2,3,4,5,6,7,8,9,34
+          field['value'].to_i
+        when 10,11
+          field['value'].to_f
+        when 32
+          t=Time.at(field['value'].to_i)
+          t + t.gmtoff
+        when 33
+          Nwsdk::Constants::NW_VARIANT_DAYS[field['value'].to_i]
+        when 128,129
+          IPAddr.new(field['value'])
+        else
+          field['value']
+      end
+    end
+    def format_timestamp(time)
+      time.getutc.strftime(Nwsdk::Constants::NW_TIME_FORMAT)
+    end
+
+    def response_successful?(response)
+      response.code==200 && response.net_http_res.content_type == 'application/json'
+    end
+
+    def get_sessionids(restq)
+      session_query=Nwsdk::Query.new(keys: ['sessionid'], condition: restq.condition, endpoint: restq.endpoint)
+      result=session_query.request
+      result.map {|r| r['sessionid']}
+    end
+
+    def get_boundary(header_val)
+      header_val.match(MULTIPART_BOUNDARY)[:msg_boundary]
+    end
+
+    def count_results(result)
+      fields=result['results'].fetch('fields',[])
+      fields.reduce({}) do |memo,field|
+        val=decode_value(field)
+        memo[val]=field['count']
+        memo
+      end
+    end
+
+    def each_multipart_response_entity(data, boundary=nil)
+
+      data.force_encoding('BINARY').split('--'+ boundary).each do |entity|
+        cleaned=entity.strip
+        next if (cleaned.match(MULTIPART_PROLOGUE) || cleaned.match(MULTIPART_END))
+        header_lines,sep,entity_data=cleaned.partition(%r{\r\n\r\n})
+        headers=Hash[header_lines.split(%r{\r\n}).map {|l| l.split(%r{: })}]
+        filename=headers['Content-Disposition'].match(ATTACHMENT_FILENAME)[:filename]
+        yield file_hash={
+          filename: filename,
+          type:     headers['Content-Type'],
+          size:     headers['Content-Length'].to_i,
+          data:     entity_data
+        }
+      end
+    end
+
+  end
+end