nwsdk 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +35 -0
- data/Gemfile +4 -0
- data/Guardfile +43 -0
- data/LICENSE +675 -0
- data/README.md +64 -0
- data/Rakefile +6 -0
- data/bin/nw +7 -0
- data/lib/nwsdk/cli.rb +210 -0
- data/lib/nwsdk/condition.rb +76 -0
- data/lib/nwsdk/constants.rb +129 -0
- data/lib/nwsdk/content.rb +63 -0
- data/lib/nwsdk/endpoint.rb +60 -0
- data/lib/nwsdk/helpers.rb +70 -0
- data/lib/nwsdk/packets.rb +56 -0
- data/lib/nwsdk/query.rb +62 -0
- data/lib/nwsdk/timeline.rb +48 -0
- data/lib/nwsdk/values.rb +38 -0
- data/lib/nwsdk/version.rb +3 -0
- data/lib/nwsdk.rb +19 -0
- data/nwsdk.gemspec +34 -0
- metadata +219 -0
data/README.md
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
# Nwsdk
|
2
|
+
|
3
|
+
Simplified wrapper + cli for NetWitness REST endpoints
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'nwsdk'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install nwsdk
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
Module documentation is non-existent. Best bet is to look at the specs and/or the cli
|
23
|
+
driver invocations.
|
24
|
+
|
25
|
+
To get up and running, invoke 'nw config' and edit ~/.nwsdk.json
|
26
|
+
|
27
|
+
The cli is mainly used from the nw command:
|
28
|
+
|
29
|
+
Commands:
|
30
|
+
nw cef CONDITIONS --loghost=LOGHOST # send cef alerts for query conditions
|
31
|
+
nw configure [$HOME/.nwsdk.json] # write out a template configuration file
|
32
|
+
nw content CONDITIONS # extract files for given query conditions
|
33
|
+
nw help [COMMAND] # Describe available commands or one specific command
|
34
|
+
nw pcap CONDITIONS # extract PCAP for given query conditions
|
35
|
+
nw query CONDITIONS # execute SDK query
|
36
|
+
nw timeline CONDITIONS # get a time-indexed histogram for conditions
|
37
|
+
nw values CONDITIONS # get value report for specific meta key
|
38
|
+
|
39
|
+
Options:
|
40
|
+
[--config=CONFIG] # JSON file with endpoint info & credentials
|
41
|
+
# Default: $HOME/.nwsdk.json
|
42
|
+
[--host=HOST] # hostname for broker or concentrator
|
43
|
+
[--port=N] # REST port for broker/concentrator
|
44
|
+
# Default: 50103
|
45
|
+
[--span=N] # max timespan in seconds
|
46
|
+
# Default: 3600
|
47
|
+
[--limit=N] # max number of sessions
|
48
|
+
# Default: 10000
|
49
|
+
[--start=START] # start time for query
|
50
|
+
# Default: $now-1h
|
51
|
+
[--end=END] # end time for query
|
52
|
+
# Default: $now-ish
|
53
|
+
|
54
|
+
|
55
|
+
|
56
|
+
## Contributing
|
57
|
+
|
58
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/ryanbreed/nwsdk.
|
59
|
+
|
60
|
+
Any fixtures/mocks/etc for the actual REST traffic would be highly welcome additions.
|
61
|
+
|
62
|
+
## License
|
63
|
+
|
64
|
+
GPLv3 (see LICENSE)
|
data/Rakefile
ADDED
data/bin/nw
ADDED
data/lib/nwsdk/cli.rb
ADDED
@@ -0,0 +1,210 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'chronic'
|
3
|
+
require 'nwsdk'
|
4
|
+
require 'cef'
|
5
|
+
require 'fileutils'
|
6
|
+
|
7
|
+
module Nwsdk
|
8
|
+
def self.setup_cli(options, where)
|
9
|
+
endpoint=Nwsdk::Endpoint.configure(options[:config])
|
10
|
+
e=case options[:end]
|
11
|
+
when nil?
|
12
|
+
options[:start] + options[:span]
|
13
|
+
else
|
14
|
+
Chronic.parse(options[:end])
|
15
|
+
end
|
16
|
+
|
17
|
+
|
18
|
+
|
19
|
+
condition=Nwsdk::Condition.new(
|
20
|
+
where: where,
|
21
|
+
start: Chronic.parse(options[:start]),
|
22
|
+
end: e,
|
23
|
+
span: options[:span]
|
24
|
+
)
|
25
|
+
{ endpoint: endpoint, condition: condition }
|
26
|
+
end
|
27
|
+
|
28
|
+
class Cli < Thor
|
29
|
+
now=Time.new
|
30
|
+
class_option :config,
|
31
|
+
default: File.join(ENV['HOME'],'.nwsdk.json'),
|
32
|
+
desc: 'JSON file with endpoint info & credentials'
|
33
|
+
class_option :host,
|
34
|
+
desc: 'hostname for broker or concentrator',
|
35
|
+
default: ENV['NW_HOST']
|
36
|
+
class_option :port,
|
37
|
+
type: :numeric,
|
38
|
+
desc: 'REST port for broker/concentrator',
|
39
|
+
default: 50103
|
40
|
+
# TODO: fix this
|
41
|
+
class_option :span,
|
42
|
+
type: :numeric,
|
43
|
+
default: 3600,
|
44
|
+
desc: 'max timespan in seconds'
|
45
|
+
class_option :limit,
|
46
|
+
type: :numeric,
|
47
|
+
default: 10000,
|
48
|
+
desc: 'max number of sessions'
|
49
|
+
# TODO: fix this
|
50
|
+
class_option :start,
|
51
|
+
desc: 'start time for query',
|
52
|
+
type: :string,
|
53
|
+
default: (now - 3600).strftime('%Y-%m-%d %H:%M:%S')
|
54
|
+
|
55
|
+
class_option :end,
|
56
|
+
desc: 'end time for query',
|
57
|
+
type: :string
|
58
|
+
class_option :debug,
|
59
|
+
type: :boolean,
|
60
|
+
default: false,
|
61
|
+
desc: 'extra info'
|
62
|
+
|
63
|
+
|
64
|
+
desc 'timeline', 'get a time-indexed histogram of sessions/packets/bytes'
|
65
|
+
option :where,
|
66
|
+
desc: 'search condition, e.g. ip.src=1.1.1.1&&service=0',
|
67
|
+
type: :string
|
68
|
+
option :flags,
|
69
|
+
type: :string,
|
70
|
+
default: 'size',
|
71
|
+
desc: 'comma-separated list of sessions, size, packets, order-ascending, order-descending'
|
72
|
+
def timeline
|
73
|
+
flags=options[:flags].split(',')
|
74
|
+
timeline=Nwsdk::Timeline.new(Nwsdk.setup_cli(options,options[:where]).merge(flags: flags))
|
75
|
+
result=timeline.request
|
76
|
+
puts JSON.pretty_generate(result)
|
77
|
+
end
|
78
|
+
|
79
|
+
desc 'values CONDITIONS', 'get value report for specific meta key'
|
80
|
+
option :size,
|
81
|
+
type: :numeric,
|
82
|
+
default: 100,
|
83
|
+
desc: 'limit to this number of results'
|
84
|
+
option :key_name,
|
85
|
+
type: :string,
|
86
|
+
default: 'service',
|
87
|
+
desc: 'meta key name'
|
88
|
+
option :where,
|
89
|
+
desc: 'search condition, e.g. ip.src=1.1.1.1&&service=0',
|
90
|
+
type: :string
|
91
|
+
option :flags,
|
92
|
+
default: 'sort-total,sessions,order-descending',
|
93
|
+
type: :string,
|
94
|
+
desc: 'comma-separated combindation of sessions, size, packets, sort-total, sort-value, order-ascending, order-descending'
|
95
|
+
def values
|
96
|
+
flags=options[:flags].split(',')
|
97
|
+
vals=Nwsdk::Values.new(Nwsdk.setup_cli(options,where=options[:where]))
|
98
|
+
vals.key_name=options[:key_name]
|
99
|
+
vals.limit=options[:size]
|
100
|
+
vals.flags=flags
|
101
|
+
result=vals.request
|
102
|
+
puts JSON.pretty_generate(result)
|
103
|
+
end
|
104
|
+
|
105
|
+
desc 'query CONDITIONS', 'execute SDK query'
|
106
|
+
option :keys, default: '*', desc: 'comma-separated list of meta keys to include'
|
107
|
+
def query(where)
|
108
|
+
nwq=Nwsdk::Query.new(Nwsdk.setup_cli(options,where))
|
109
|
+
nwq.keys=options[:keys].split(',')
|
110
|
+
result=nwq.request
|
111
|
+
puts JSON.pretty_generate(result)
|
112
|
+
end
|
113
|
+
|
114
|
+
desc 'content CONDITIONS', 'extract files for given query conditions'
|
115
|
+
option :dir, default: 'tmp', desc: 'output directory'
|
116
|
+
option :exclude, default: '', desc: 'comma-separated list of file extensions to exclude'
|
117
|
+
option :include, default: '', desc: 'include only this comma-separated list of extensions'
|
118
|
+
def content(where)
|
119
|
+
content=Nwsdk::Content.new(Nwsdk.setup_cli(options,where))
|
120
|
+
content.output_dir=options[:dir]
|
121
|
+
incl=options[:include].split(',')
|
122
|
+
excl=options[:exclude].split(',')
|
123
|
+
content.include_types=incl unless incl==[]
|
124
|
+
content.exclude_types=excl unless excl==[]
|
125
|
+
content.each_session_file do |file|
|
126
|
+
FileUtils.mkdir_p(options[:dir]) unless Dir.exist?(options[:dir])
|
127
|
+
outf=File.join(options[:dir],file[:filename])
|
128
|
+
STDERR.puts "writing #{outf}"
|
129
|
+
File.open(outf,'w') {|f| f.write(file[:data]) }
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
desc 'pcap CONDITIONS', 'extract PCAP for given query conditions'
|
134
|
+
option :prefix, default: 'pcap_%08d' % $$, desc: 'file name prefix'
|
135
|
+
option :group, type: :numeric, desc: 'max sessions per pcap file', default: 1000
|
136
|
+
def pcap(where)
|
137
|
+
p=Nwsdk::Packets.new(Nwsdk.setup_cli(options,where))
|
138
|
+
p.group=options[:group]
|
139
|
+
p.file_prefix=options[:prefix]
|
140
|
+
p.each_pcap_group do |g|
|
141
|
+
STDERR.puts "Writing #{g[:filename]}"
|
142
|
+
File.open(g[:filename],'w') {|f| f.write(g[:data])}
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
desc 'cef CONDITIONS', 'send cef alerts for query conditions'
|
147
|
+
option :name,
|
148
|
+
desc: 'name of event',
|
149
|
+
default: 'cef alert'
|
150
|
+
option :loghost,
|
151
|
+
required: true,
|
152
|
+
desc: 'syslog destination'
|
153
|
+
option :logport,
|
154
|
+
default: 514,
|
155
|
+
type: :numeric,
|
156
|
+
desc: 'syslog UDP port'
|
157
|
+
def cef(where)
|
158
|
+
nwq = Nwsdk::Query.new(Nwsdk.setup_cli(options,where))
|
159
|
+
nwq.keys = ['*']
|
160
|
+
result = nwq.request
|
161
|
+
|
162
|
+
mapping = nwq.endpoint.config['cef_mapping']
|
163
|
+
|
164
|
+
sender = case nwq.endpoint.loghost
|
165
|
+
when nil
|
166
|
+
CEF::UDPSender.new(options[:loghost],options[:logport])
|
167
|
+
else
|
168
|
+
CEF::UDPSender.new(*nwq.endpoint.loghost)
|
169
|
+
end
|
170
|
+
|
171
|
+
result.each do |res|
|
172
|
+
event=CEF::Event.new
|
173
|
+
event_fields=mapping.keys & res.keys
|
174
|
+
event_fields.each do |field|
|
175
|
+
event.send('%s=' % mapping[field],res[field].to_s)
|
176
|
+
end
|
177
|
+
nwq.endpoint.config['cef_static_fields'].each {|k,v| event.send('%s='%k,v)}
|
178
|
+
event.name=options[:name]
|
179
|
+
event.endTime=(res['time'].to_i * 1000).to_s
|
180
|
+
puts event.to_s
|
181
|
+
sender.emit(event)
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
config=File.join(ENV['HOME'],'.nwsdk.json')
|
186
|
+
desc 'configure /path/to/config.json', 'write out a template configuration file'
|
187
|
+
option :user,
|
188
|
+
default: 'admin',
|
189
|
+
desc: 'username with sdk query rights'
|
190
|
+
option :pass,
|
191
|
+
default: 'netwitness',
|
192
|
+
desc: 'password'
|
193
|
+
option :loghost,
|
194
|
+
desc: 'syslog destination for cef alerts'
|
195
|
+
option :logport,
|
196
|
+
default: 514,
|
197
|
+
type: :numeric,
|
198
|
+
desc: 'syslog UDP port'
|
199
|
+
def configure(path=File.join(ENV['HOME'],'.nwsdk.json'))
|
200
|
+
conf=Nwsdk::Constants::DEFAULT_CONFIG.dup
|
201
|
+
conf['endpoint']['host']=options[:host] unless options[:host].nil?
|
202
|
+
conf['endpoint']['port']=options[:port]
|
203
|
+
conf['endpoint']['user']=options[:user]
|
204
|
+
conf['endpoint']['pass']=options[:pass]
|
205
|
+
conf['syslog']['loghost']=options[:loghost] unless options[:loghost].nil?
|
206
|
+
conf['syslog']['logport']=options[:logport]
|
207
|
+
File.open(path,'w') {|f| f.write JSON.pretty_generate(conf) }
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
module Nwsdk
|
2
|
+
class Condition
|
3
|
+
include Helpers
|
4
|
+
attr_accessor :where, :start, :end, :span, :id1, :id2, :tz_offset
|
5
|
+
|
6
|
+
def initialize(*args)
|
7
|
+
Hash[*args].each {|k,v| self.send("%s="%k, v)}
|
8
|
+
init = Time.new
|
9
|
+
|
10
|
+
# TODO: this is hella ugly, mane
|
11
|
+
if @end.nil?
|
12
|
+
@span ||= 3600
|
13
|
+
if @start.nil?
|
14
|
+
@end=init
|
15
|
+
@start=@end - @span
|
16
|
+
else
|
17
|
+
@end = @start + @span
|
18
|
+
end
|
19
|
+
else
|
20
|
+
if @start.nil?
|
21
|
+
@span ||= 3600
|
22
|
+
@start = @end - @span
|
23
|
+
else
|
24
|
+
@span = @end - @start
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
@tz_offset = @start.gmtoff
|
29
|
+
end
|
30
|
+
|
31
|
+
def format(use_time: true)
|
32
|
+
formatted_where=case where
|
33
|
+
when nil
|
34
|
+
""
|
35
|
+
else
|
36
|
+
sprintf('(%s)',where)
|
37
|
+
end
|
38
|
+
if use_time
|
39
|
+
if formatted_where==""
|
40
|
+
sprintf('time=%s', format_time_range)
|
41
|
+
else
|
42
|
+
sprintf('%s&&time=%s',formatted_where,format_time_range)
|
43
|
+
end
|
44
|
+
else
|
45
|
+
formatted_where
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def format_time_range
|
50
|
+
sprintf("'%s'-'%s'",
|
51
|
+
format_timestamp(@start),
|
52
|
+
format_timestamp(@end)
|
53
|
+
)
|
54
|
+
end
|
55
|
+
|
56
|
+
# alias for #start
|
57
|
+
|
58
|
+
def time1=(t)
|
59
|
+
@start=t
|
60
|
+
end
|
61
|
+
|
62
|
+
def time1
|
63
|
+
@start
|
64
|
+
end
|
65
|
+
|
66
|
+
# alias for #end
|
67
|
+
|
68
|
+
def time2=(t)
|
69
|
+
@end=t
|
70
|
+
end
|
71
|
+
|
72
|
+
def time2
|
73
|
+
@end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,129 @@
|
|
1
|
+
module Nwsdk
|
2
|
+
module Constants
|
3
|
+
NW_TIME_FORMAT = '%Y-%b-%d %H:%M:%S'
|
4
|
+
NW_VARIANT_DAYS = %w{
|
5
|
+
Sunday
|
6
|
+
Monday
|
7
|
+
Tuesday
|
8
|
+
Wednesday
|
9
|
+
Thursday
|
10
|
+
Friday
|
11
|
+
Saturday
|
12
|
+
}
|
13
|
+
#######
|
14
|
+
#
|
15
|
+
# Nwsdk Content Render Types
|
16
|
+
#
|
17
|
+
#######
|
18
|
+
|
19
|
+
NW_CONTENT_TYPE_AUTO = 0, # Auto Select HTML Content View
|
20
|
+
NW_CONTENT_TYPE_DETAILS = 1, # HTML Meta Details View
|
21
|
+
NW_CONTENT_TYPE_TEXT = 2, # HTML Text View
|
22
|
+
NW_CONTENT_TYPE_HEX = 3, # HTML Hex View
|
23
|
+
NW_CONTENT_TYPE_PACKETS = 4, # HTML Packet View
|
24
|
+
NW_CONTENT_TYPE_MAIL = 5, # HTML Mail View
|
25
|
+
NW_CONTENT_TYPE_WEB = 6, # HTML Web Page View
|
26
|
+
NW_CONTENT_TYPE_VOIP = 7, # HTML VOIP View
|
27
|
+
NW_CONTENT_TYPE_IM = 8, # HTML IM View
|
28
|
+
NW_CONTENT_TYPE_FILES = 9, # HTML Listing of all files found in session
|
29
|
+
NW_CONTENT_TYPE_PCAP = 100, # Pcap Packet File
|
30
|
+
NW_CONTENT_TYPE_RAW = 102, # Raw Content
|
31
|
+
NW_CONTENT_TYPE_XML = 103, # Meta XML File
|
32
|
+
NW_CONTENT_TYPE_CSV = 104, # Meta Comma Separated File
|
33
|
+
NW_CONTENT_TYPE_TXT = 105, # Meta Tab Separated File
|
34
|
+
NW_CONTENT_TYPE_NWD = 106, # Netwitness Data File
|
35
|
+
NW_CONTENT_TYPE_FILE_EXTRACTOR = 107, # Extract files from common protocols
|
36
|
+
NW_CONTENT_TYPE_LOGS = 108, # Log extract (captured logs, LF delimited)
|
37
|
+
NW_CONTENT_TYPE_PROTOBUF = 109 # Content of the NWD as a Google Protocol Buffer object
|
38
|
+
|
39
|
+
######
|
40
|
+
#
|
41
|
+
# Nwsdk Content Render Flags
|
42
|
+
#
|
43
|
+
######
|
44
|
+
|
45
|
+
NW_CONTENT_FLAG_STREAM1 = 0x00001, # Return only request stream
|
46
|
+
# content.
|
47
|
+
NW_CONTENT_FLAG_STREAM2 = 0x00002, # Return only response stream
|
48
|
+
# content.
|
49
|
+
NW_CONTENT_FLAG_SINGLE_COLUMN = 0x00004, # Format generated web page as
|
50
|
+
# a single column with requests
|
51
|
+
# and responses interleaved.
|
52
|
+
NW_CONTENT_FLAG_PACKET_PAYLOAD = 0x00008, # Include only session payload.
|
53
|
+
NW_CONTENT_FLAG_DECODEAS_EBCDIC = 0x00010, # Convert session payload from EBCDIC to ASCII.
|
54
|
+
NW_CONTENT_FLAG_DO_NOT_EMBED = 0x00020, # Do not embed application/
|
55
|
+
# audio/video traffic into the
|
56
|
+
# generated web page.
|
57
|
+
NW_CONTENT_FLAG_UNCOMPRESS_TEXT = 0x00040, # Unzip web content in text view.
|
58
|
+
NW_CONTENT_FLAG_DECODE_SSL = 0x00080, # Attempt to decrypt SSL
|
59
|
+
# sessions if the encryption
|
60
|
+
# key is provided.
|
61
|
+
NW_CONTENT_FLAG_STRIP_STYLE_TAGS = 0x00100, # Removes all <style> tags from the original
|
62
|
+
# html document.
|
63
|
+
NW_CONTENT_FLAG_IGNORE_CACHE = 0x01000, # Ignore any content in cache
|
64
|
+
# and requery, affects only
|
65
|
+
# current request.
|
66
|
+
NW_CONTENT_FLAG_NO_EMBEDDED_EXE = 0x02000, # Do not look for or extract
|
67
|
+
# hidden/embedded PE files
|
68
|
+
# when performing file
|
69
|
+
# extraction.
|
70
|
+
NW_CONTENT_FLAG_INCLUDE_DUPS = 0x04000, # Include packets otherwise removed by assembly
|
71
|
+
NW_CONTENT_FLAG_INCLUDE_HEADERS = 0x08000, # Include packet header meta information
|
72
|
+
NW_CONTENT_FLAG_CAPTURE_ORDER = 0x10000 # Do not assemble packets, return in capture order
|
73
|
+
|
74
|
+
######
|
75
|
+
#
|
76
|
+
# d.'e'fault config
|
77
|
+
#
|
78
|
+
#####
|
79
|
+
DEFAULT_CONFIG = {
|
80
|
+
"endpoint"=> {
|
81
|
+
"user"=>"admin",
|
82
|
+
"pass"=>"netwitness",
|
83
|
+
"host"=>"broker.local",
|
84
|
+
"port"=>"50103"
|
85
|
+
},
|
86
|
+
"syslog"=>{
|
87
|
+
"loghost"=>"loghost.local",
|
88
|
+
"logport"=>514
|
89
|
+
},
|
90
|
+
"cef_static_fields"=> {
|
91
|
+
"deviceVendor"=>"ERCOT",
|
92
|
+
"deviceProduct"=>"nwsdk",
|
93
|
+
"deviceCustomString1Label"=>"threat.desc",
|
94
|
+
"deviceCustomString2Label"=>"threat.source",
|
95
|
+
"deviceCustomString3Label"=>"threat.category",
|
96
|
+
"deviceCustomNumber1Label"=>"asn.src",
|
97
|
+
"deviceCustomNumber2Label"=>"asn.dst"
|
98
|
+
},
|
99
|
+
"cef_mapping" => {
|
100
|
+
"action"=>"deviceAction",
|
101
|
+
"alias.host"=>"destinationProcessName",
|
102
|
+
"asn.dst"=>"deviceCustomNumber2",
|
103
|
+
"asn.src"=>"deviceCustomNumber1",
|
104
|
+
"did"=>"deviceHostName",
|
105
|
+
"directory"=>"filePath",
|
106
|
+
"domain.dst"=>"destinationHostName",
|
107
|
+
"eth.dst"=>"destinationMacAddress",
|
108
|
+
"eth.src"=>"sourceMacAddress",
|
109
|
+
"filename"=>"fileName",
|
110
|
+
"ip.dst"=>"destinationAddress",
|
111
|
+
"ip.proto"=>"transportProtocol",
|
112
|
+
"ip.src"=>"sourceAddress",
|
113
|
+
"risk.warning"=>"name",
|
114
|
+
"risk.suspicious"=>"name",
|
115
|
+
"service"=>"applicationProtocol",
|
116
|
+
"sessionid"=>"externalId",
|
117
|
+
"size"=>"bytesIn",
|
118
|
+
"tcp.dstport"=>"destinationPort",
|
119
|
+
"tcp.srcport"=>"sourcePort",
|
120
|
+
"threat.desc"=>"deviceCustomString1",
|
121
|
+
"threat.source"=>"deviceCustomString2",
|
122
|
+
"threat.category"=>"deviceCustomString3",
|
123
|
+
"udp.dstport"=>"destinationPort",
|
124
|
+
"udp.srcport"=>"sourcePort",
|
125
|
+
"username"=>"destinationUserName"
|
126
|
+
}
|
127
|
+
}
|
128
|
+
end
|
129
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
module Nwsdk
|
2
|
+
class Content
|
3
|
+
include Helpers
|
4
|
+
include Constants
|
5
|
+
|
6
|
+
attr_accessor :output_dir, :include_types, :exclude_types,
|
7
|
+
:condition, :endpoint
|
8
|
+
|
9
|
+
def initialize(*args)
|
10
|
+
Hash[*args].each {|k,v| self.send('%s='%k, v)}
|
11
|
+
@include_types ||= []
|
12
|
+
@exclude_types ||= []
|
13
|
+
@output_dir ||= 'tmp'
|
14
|
+
end
|
15
|
+
|
16
|
+
def build_request(session)
|
17
|
+
endpoint.get_request(
|
18
|
+
path: 'sdk/content',
|
19
|
+
params: build_params(session)
|
20
|
+
)
|
21
|
+
end
|
22
|
+
|
23
|
+
def each_session
|
24
|
+
sessions=get_sessionids(self)
|
25
|
+
sessions.each do |sessionid|
|
26
|
+
begin
|
27
|
+
response=build_request(sessionid).execute
|
28
|
+
rescue RestClient::Gone
|
29
|
+
next
|
30
|
+
end
|
31
|
+
next unless response.code==200
|
32
|
+
yield response
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def each_session_file
|
37
|
+
each_session do |response|
|
38
|
+
content=response.body.encode('BINARY')
|
39
|
+
next unless response.headers.has_key? :content_type
|
40
|
+
boundary=get_boundary(response.headers[:content_type])
|
41
|
+
each_multipart_response_entity(content,boundary) do |file|
|
42
|
+
yield file
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def build_params(session)
|
48
|
+
send_params={
|
49
|
+
session: session,
|
50
|
+
maxSize: 0,
|
51
|
+
render: NW_CONTENT_TYPE_FILE_EXTRACTOR
|
52
|
+
}
|
53
|
+
unless include_types.empty?
|
54
|
+
send_params[:includeFileTypes]=include_types.join(';')
|
55
|
+
end
|
56
|
+
unless exclude_types.empty?
|
57
|
+
send_params[:excludeFileTypes]=exclude_types.join(';')
|
58
|
+
end
|
59
|
+
send_params
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module Nwsdk
|
2
|
+
class Endpoint
|
3
|
+
attr_accessor :host, :port, :user, :pass, :config, :save_request
|
4
|
+
def self.configure(conf=File.join(ENV["HOME"],".nwsdk.json"))
|
5
|
+
config=JSON.parse(File.read(conf))
|
6
|
+
endpoint=Nwsdk::Endpoint.new(config["endpoint"])
|
7
|
+
endpoint.config=config
|
8
|
+
endpoint
|
9
|
+
end
|
10
|
+
|
11
|
+
def initialize(*args)
|
12
|
+
Hash[*args].each {|k,v| self.send("%s=" % k, v) }
|
13
|
+
@port ||= 50103
|
14
|
+
@save_request||=false
|
15
|
+
yield self if block_given?
|
16
|
+
self
|
17
|
+
end
|
18
|
+
|
19
|
+
def type=(type_sym)
|
20
|
+
@port=case type_sym
|
21
|
+
when :broker
|
22
|
+
50103
|
23
|
+
when :decoder
|
24
|
+
50104
|
25
|
+
when :concentrator
|
26
|
+
50105
|
27
|
+
else
|
28
|
+
raise ArgumentError, "invalid endpoint type #{type_sym.to_s}"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def uri(path='sdk')
|
33
|
+
sprintf('https://%s:%d/%s', host, port, path)
|
34
|
+
end
|
35
|
+
|
36
|
+
def get_request(path: 'sdk', params: {})
|
37
|
+
req=RestClient::Request.new(
|
38
|
+
method: :get,
|
39
|
+
url: uri(path),
|
40
|
+
user: user,
|
41
|
+
password: pass,
|
42
|
+
read_timenout: nil,
|
43
|
+
verify_ssl: OpenSSL::SSL::VERIFY_NONE,
|
44
|
+
payload: params,
|
45
|
+
headers: {
|
46
|
+
"Accept-Encoding" => :deflate,
|
47
|
+
accept: :json,
|
48
|
+
}
|
49
|
+
)
|
50
|
+
end
|
51
|
+
|
52
|
+
def loghost
|
53
|
+
if config.has_key?("syslog")
|
54
|
+
[ config["syslog"]["loghost"], config["syslog"]["logport"]]
|
55
|
+
else
|
56
|
+
nil
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
module Nwsdk
|
2
|
+
module Helpers
|
3
|
+
MULTIPART_BOUNDARY=%r{\Amultipart/mixed; boundary="(?<msg_boundary>.+)"\z}
|
4
|
+
MULTIPART_PROLOGUE=%r{\AThis is a message with multiple parts in MIME format.}
|
5
|
+
MULTIPART_END=%r{\A--\z}
|
6
|
+
ATTACHMENT_FILENAME=%r{\Aattachment; filename="(?<filename>.+)"\z}
|
7
|
+
|
8
|
+
def decode_value(field)
|
9
|
+
case field['format']
|
10
|
+
when 1,2,3,4,5,6,7,8,9,34
|
11
|
+
field['value'].to_i
|
12
|
+
when 10,11
|
13
|
+
field['value'].to_f
|
14
|
+
when 32
|
15
|
+
t=Time.at(field['value'].to_i)
|
16
|
+
t + t.gmtoff
|
17
|
+
when 33
|
18
|
+
Nwsdk::Constants::NW_VARIANT_DAYS[field['value'].to_i]
|
19
|
+
when 128,129
|
20
|
+
IPAddr.new(field['value'])
|
21
|
+
else
|
22
|
+
field['value']
|
23
|
+
end
|
24
|
+
end
|
25
|
+
def format_timestamp(time)
|
26
|
+
time.getutc.strftime(Nwsdk::Constants::NW_TIME_FORMAT)
|
27
|
+
end
|
28
|
+
|
29
|
+
def response_successful?(response)
|
30
|
+
response.code==200 && response.net_http_res.content_type == 'application/json'
|
31
|
+
end
|
32
|
+
|
33
|
+
def get_sessionids(restq)
|
34
|
+
session_query=Nwsdk::Query.new(keys: ['sessionid'], condition: restq.condition, endpoint: restq.endpoint)
|
35
|
+
result=session_query.request
|
36
|
+
result.map {|r| r['sessionid']}
|
37
|
+
end
|
38
|
+
|
39
|
+
def get_boundary(header_val)
|
40
|
+
header_val.match(MULTIPART_BOUNDARY)[:msg_boundary]
|
41
|
+
end
|
42
|
+
|
43
|
+
def count_results(result)
|
44
|
+
fields=result['results'].fetch('fields',[])
|
45
|
+
fields.reduce({}) do |memo,field|
|
46
|
+
val=decode_value(field)
|
47
|
+
memo[val]=field['count']
|
48
|
+
memo
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def each_multipart_response_entity(data, boundary=nil)
|
53
|
+
|
54
|
+
data.force_encoding('BINARY').split('--'+ boundary).each do |entity|
|
55
|
+
cleaned=entity.strip
|
56
|
+
next if (cleaned.match(MULTIPART_PROLOGUE) || cleaned.match(MULTIPART_END))
|
57
|
+
header_lines,sep,entity_data=cleaned.partition(%r{\r\n\r\n})
|
58
|
+
headers=Hash[header_lines.split(%r{\r\n}).map {|l| l.split(%r{: })}]
|
59
|
+
filename=headers['Content-Disposition'].match(ATTACHMENT_FILENAME)[:filename]
|
60
|
+
yield file_hash={
|
61
|
+
filename: filename,
|
62
|
+
type: headers['Content-Type'],
|
63
|
+
size: headers['Content-Length'].to_i,
|
64
|
+
data: entity_data
|
65
|
+
}
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|