dorothy2 0.0.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +30 -6
- data/TODO +21 -0
- data/bin/dorothy_start +7 -6
- data/bin/dparser_start +13 -1
- data/etc/ddl/dorothive.ddl +2 -31
- data/lib/doroParser.rb +30 -23
- data/lib/dorothy2/BFM.rb +5 -13
- data/lib/dorothy2/do-utils.rb +4 -5
- data/lib/dorothy2/version.rb +1 -1
- data/lib/dorothy2.rb +5 -8
- data/lib/mu/xtractr/about.rb +57 -0
- data/lib/mu/xtractr/content.rb +68 -0
- data/lib/mu/xtractr/field.rb +178 -0
- data/lib/mu/xtractr/flow.rb +162 -0
- data/lib/mu/xtractr/flows.rb +118 -0
- data/lib/mu/xtractr/host.rb +87 -0
- data/lib/mu/xtractr/packet.rb +138 -0
- data/lib/mu/xtractr/packets.rb +122 -0
- data/lib/mu/xtractr/service.rb +77 -0
- data/lib/mu/xtractr/stream/http.rb +103 -0
- data/lib/mu/xtractr/stream.rb +132 -0
- data/lib/mu/xtractr/term.rb +73 -0
- data/lib/mu/xtractr/test/stream/tc_http.rb +53 -0
- data/lib/mu/xtractr/test/tc_field.rb +140 -0
- data/lib/mu/xtractr/test/tc_flow.rb +79 -0
- data/lib/mu/xtractr/test/tc_flows.rb +94 -0
- data/lib/mu/xtractr/test/tc_host.rb +116 -0
- data/lib/mu/xtractr/test/tc_packet.rb +110 -0
- data/lib/mu/xtractr/test/tc_packets.rb +84 -0
- data/lib/mu/xtractr/test/tc_service.rb +66 -0
- data/lib/mu/xtractr/test/tc_stream.rb +56 -0
- data/lib/mu/xtractr/test/tc_term.rb +59 -0
- data/lib/mu/xtractr/test/tc_views.rb +118 -0
- data/lib/mu/xtractr/test/tc_xtractr.rb +151 -0
- data/lib/mu/xtractr/test/test.rb +19 -0
- data/lib/mu/xtractr/views.rb +204 -0
- data/lib/mu/xtractr.rb +257 -0
- metadata +32 -4
data/lib/mu/xtractr.rb
ADDED
@@ -0,0 +1,257 @@
|
|
1
|
+
# "THE BEER-WARE LICENSE" (Revision 42):
|
2
|
+
# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this
|
3
|
+
# notice you can do whatever you want with this stuff. If we meet some day,
|
4
|
+
# and you think this stuff is worth it, you can buy us a beer in return.
|
5
|
+
#
|
6
|
+
# All about pcapr
|
7
|
+
# * http://www.pcapr.net
|
8
|
+
# * http://groups.google.com/group/pcapr-forum
|
9
|
+
# * http://twitter.com/pcapr
|
10
|
+
#
|
11
|
+
# Mu Dynamics
|
12
|
+
# * http://www.mudynamics.com
|
13
|
+
# * http://labs.mudynamics.com
|
14
|
+
|
15
|
+
require 'mu/xtractr/about'
|
16
|
+
require 'mu/xtractr/content'
|
17
|
+
require 'mu/xtractr/field'
|
18
|
+
require 'mu/xtractr/flow'
|
19
|
+
require 'mu/xtractr/flows'
|
20
|
+
require 'mu/xtractr/host'
|
21
|
+
require 'mu/xtractr/packet'
|
22
|
+
require 'mu/xtractr/packets'
|
23
|
+
require 'mu/xtractr/service'
|
24
|
+
require 'mu/xtractr/stream'
|
25
|
+
require 'mu/xtractr/term'
|
26
|
+
require 'mu/xtractr/views'
|
27
|
+
|
28
|
+
module Mu # :nodoc:
|
29
|
+
# = http://www.pcapr.net/static/image/favicon.png Mu::Xtractr
|
30
|
+
# :main: Mu::Xtractr
|
31
|
+
#
|
32
|
+
# This gem is Ruby front-end to the RESTful API that <b>xtractr[http://www.pcapr.net/xtractr]</b>
|
33
|
+
# provides. We primarily use this for unit testing xtractr's API, but on
|
34
|
+
# its own this gem provides for a powerful programmable interface into
|
35
|
+
# xtractr and is a super fast way to extract information out of large pcaps.
|
36
|
+
#
|
37
|
+
# = Getting Started
|
38
|
+
# First download <b>xtractr</b> from http://www.pcapr.net/xtractr. Follow the
|
39
|
+
# instructions to index your pcap. Finally run xtractr in browse mode and then
|
40
|
+
# you can hang out in IRB poking around flows and packets.
|
41
|
+
#
|
42
|
+
# = Examples
|
43
|
+
# You can run the xtractr-gem from within IRB which makes it a fun interactive
|
44
|
+
# network forensics tool. Make sure you are running the xtractr binary in
|
45
|
+
# browse mode. Turning on auto-completion in IRB also makes it easier to try out
|
46
|
+
# different things and interactively experiment with the API:
|
47
|
+
#
|
48
|
+
# $ irb -rirb/completion -rmu/xtractr
|
49
|
+
#
|
50
|
+
# All of the examples below work off the test/test.pcap bundled with the gem.
|
51
|
+
# We'll also assume that you've done this at the start of the IRB session:
|
52
|
+
#
|
53
|
+
# irb> xtractr = Mu::Xtractr.new
|
54
|
+
#
|
55
|
+
# <b>Top DNS query names</b>
|
56
|
+
#
|
57
|
+
# We first pull out all DNS flows and then map/reduce the unique values of
|
58
|
+
# the <em>dns.qry.name</em>.
|
59
|
+
# irb> xtractr.flows('flow.service:DNS').values('dns.qry.name')
|
60
|
+
#
|
61
|
+
# <b>Services used by the top talker (based on bytes sent/received)</b>
|
62
|
+
#
|
63
|
+
# We first sum the total number of bytes using the src address as the key. The
|
64
|
+
# sum function returns the matches sorted by the #bytes. We then use the first
|
65
|
+
# object (the top talker) to in turn map/reduce the unique list of services
|
66
|
+
# supported by it.
|
67
|
+
# irb> xtractr.flows.sum('flow.src', 'flow.bytes').first.count('flow.service')
|
68
|
+
#
|
69
|
+
# <b>Generating #new pcaps based on search criteria</b>
|
70
|
+
#
|
71
|
+
# We first get a list of the unique HTTP methods in the index and then for each of
|
72
|
+
# methods, query for all the packets and then save them into a new pcap.
|
73
|
+
# irb> xtractr.packets.count('http.request.method').each { |c| c.packets.save("#{c.value}.pcap") }
|
74
|
+
#
|
75
|
+
#--
|
76
|
+
# rdoc --exclude test --force-update --inline-source --op mu/xtractr/doc --main Mu::Xtractr
|
77
|
+
class Xtractr
|
78
|
+
# Return the IP address of the xtractr instance
|
79
|
+
attr_reader :address
|
80
|
+
|
81
|
+
# Return the listening port of the xtractr instance
|
82
|
+
attr_reader :port
|
83
|
+
|
84
|
+
# Relative URL
|
85
|
+
attr_reader :relurl
|
86
|
+
|
87
|
+
# Create a new instance to connect to the xtractr binary using a
|
88
|
+
# url.
|
89
|
+
# Xtractr.create 'http://some.host:8080/'
|
90
|
+
def self.create url
|
91
|
+
uri = URI.parse url
|
92
|
+
self.new uri.host, uri.port, uri.path
|
93
|
+
end
|
94
|
+
|
95
|
+
# Create a new instance to connect to the xtractr binary running in
|
96
|
+
# browse mode.
|
97
|
+
# Xtractr.new
|
98
|
+
# Xtractr.new 'localhost', 8080
|
99
|
+
def initialize address='localhost', port=8080, relurl=nil
|
100
|
+
@address = address
|
101
|
+
@port = port
|
102
|
+
@relurl = relurl || '/'
|
103
|
+
@relurl << '/' if @relurl[-1,1] != '/'
|
104
|
+
#unless about.version =~ /^4\.5\.(svn|41604)$/
|
105
|
+
# puts "boh"
|
106
|
+
# puts "xtractr version #{about.version} out of date!"
|
107
|
+
# puts "please download a new one from http://www.pcapr.net/xtractr"
|
108
|
+
# raise
|
109
|
+
#end
|
110
|
+
end
|
111
|
+
|
112
|
+
# Fetch the meta data about the index. This includes information about
|
113
|
+
# the total number of packets, flows as well as the duration of the entire
|
114
|
+
# set of pcaps in the index.
|
115
|
+
# xtractr.about
|
116
|
+
def about
|
117
|
+
@about ||= About.new json('api/about')
|
118
|
+
end
|
119
|
+
|
120
|
+
# Fetch the list of fields in the index. The fields are only available
|
121
|
+
# if the <em>--mode forensics</em> was used during the indexing process.
|
122
|
+
# xtractr.fields
|
123
|
+
# xtractr.fields /^http/
|
124
|
+
# xtractr.fields 'http.server'
|
125
|
+
def fields regex=nil
|
126
|
+
regex = Regexp.new(regex, Regexp::IGNORECASE) if regex.is_a? String
|
127
|
+
result = (@fields ||= json 'api/fields')
|
128
|
+
result = result.select { |name| name =~ regex } if regex
|
129
|
+
return result.map { |name| Field.new self, name }
|
130
|
+
end
|
131
|
+
|
132
|
+
# Fetch a field of the given name.
|
133
|
+
# xtractr.field 'http.server'
|
134
|
+
def field name
|
135
|
+
obj = fields.find { |f| f.name == name }
|
136
|
+
raise ArgumentError, "Unknown field #{name}" if not obj
|
137
|
+
return obj
|
138
|
+
end
|
139
|
+
|
140
|
+
# Fetch the list of hosts in the index. The optional regex (or String)
|
141
|
+
# can be used to filter the hosts list.
|
142
|
+
# xtractr.hosts
|
143
|
+
# xtractr.hosts /192.168/
|
144
|
+
# xtractr.hosts '10.10'
|
145
|
+
def hosts regex=nil
|
146
|
+
regex = Regexp.new(regex, Regexp::IGNORECASE) if regex.is_a? String
|
147
|
+
result = (@hosts ||= json 'api/hosts')
|
148
|
+
rows = result['rows']
|
149
|
+
rows = rows.select { |row| row['name'] =~ regex } if regex
|
150
|
+
return rows.map { |row| Host.new self, row['name'] }
|
151
|
+
end
|
152
|
+
|
153
|
+
# Fetch a host of the given address.
|
154
|
+
# xtractr.host '192.168.1.1'
|
155
|
+
def host address
|
156
|
+
obj = hosts.find { |h| h.address == address }
|
157
|
+
raise ArgumentError, "Unknown host #{address}" if not obj
|
158
|
+
return obj
|
159
|
+
end
|
160
|
+
|
161
|
+
# Fetch the list of services in the index. The optional regex (or String)
|
162
|
+
# can be used to filter the services lists.
|
163
|
+
# xtractr.services
|
164
|
+
# xtractr.services /http/
|
165
|
+
# xtractr.services 'sip'
|
166
|
+
def services regex=nil
|
167
|
+
regex = Regexp.new(regex, Regexp::IGNORECASE) if regex.is_a? String
|
168
|
+
result = (@services ||= json 'api/services')
|
169
|
+
rows = result['rows']
|
170
|
+
rows = rows.select { |row| row['name'] =~ regex } if regex
|
171
|
+
return rows.map { |row| Service.new self, row['name'] }
|
172
|
+
end
|
173
|
+
|
174
|
+
# Fetch a service of the given name.
|
175
|
+
# xtractr.service 'dns'
|
176
|
+
def service name
|
177
|
+
obj = services.find { |s| s.name.downcase == name.downcase }
|
178
|
+
raise ArgumentError, "Unknown service #{name}" if not obj
|
179
|
+
return obj
|
180
|
+
end
|
181
|
+
|
182
|
+
# Return an iterator that can yield each flow that matched the query. If
|
183
|
+
# <em>q</em> is a Range, then it's used to extract the set of flows that match
|
184
|
+
# all of those ids.
|
185
|
+
# xtractr.flows.each { |flow| ... }
|
186
|
+
# xtractr.flows(1..10).each { |flow| ... }
|
187
|
+
# xtractr.flows("flow.src:192.168.1.1").each { |flow| ... }
|
188
|
+
def flows(q='*') # :yields: flow
|
189
|
+
if q.is_a? Range
|
190
|
+
first = q.first
|
191
|
+
last = q.last
|
192
|
+
last -= 1 if q.exclude_end?
|
193
|
+
q = "flow.id:[#{first} #{last}]"
|
194
|
+
end
|
195
|
+
return Flows.new(self, :q => q)
|
196
|
+
end
|
197
|
+
|
198
|
+
# Return the id'th flow from the index.
|
199
|
+
# xtractr.flow 1
|
200
|
+
def flow id
|
201
|
+
result = json "api/flows", :start => id, :limit => id
|
202
|
+
rows = result['rows']
|
203
|
+
raise ArgumentError, "Unknown flow #{id}" if rows.empty?
|
204
|
+
return Flow.new(self, rows[0])
|
205
|
+
end
|
206
|
+
|
207
|
+
# Return an iterator that can yield each packet that matched the query. If
|
208
|
+
# <em>q</em> is a Range, then it's used to extract the set of packets that match
|
209
|
+
# all of those ids.
|
210
|
+
# xtractr.packets.each { |pkt| ... }
|
211
|
+
# xtractr.packets(5..32).each { |pkt| ... }
|
212
|
+
# xtractr.packets("http.user.agent:mozilla").each { |pkt| ... }
|
213
|
+
def packets(q='*') # :yields: packet
|
214
|
+
if q.is_a? Range
|
215
|
+
first = q.first
|
216
|
+
last = q.last
|
217
|
+
last -= 1 if q.exclude_end?
|
218
|
+
q = "pkt.id:[#{first} #{last}]"
|
219
|
+
end
|
220
|
+
return Packets.new(self, :q => q)
|
221
|
+
end
|
222
|
+
|
223
|
+
# Return the id'th packet from the index.
|
224
|
+
# xtractr.packet 1
|
225
|
+
def packet id
|
226
|
+
result = json "api/packets", :start => id, :limit => id
|
227
|
+
rows = result['rows']
|
228
|
+
raise ArgumentError, "Unknown packet #{id}" if rows.empty?
|
229
|
+
return Packet.new(self, rows[0])
|
230
|
+
end
|
231
|
+
|
232
|
+
# Fetch the URL with the GET parameters and interpret the response body
|
233
|
+
# as a JSON object
|
234
|
+
def json url, opts={} # :nodoc:
|
235
|
+
res = get url, opts
|
236
|
+
js = JSON.parse(res)
|
237
|
+
raise ArgumentError, js['reason'] if js.is_a?(Hash) and js['error']
|
238
|
+
return js
|
239
|
+
end
|
240
|
+
|
241
|
+
# Fetch the URL and return the response body, as is
|
242
|
+
def get url, opts={} # :nodoc:
|
243
|
+
_url = relurl + url
|
244
|
+
if opts.size
|
245
|
+
_url << '?'
|
246
|
+
_url << opts.keys.map { |key| key.to_s + '=' + opts[key].to_s }.join('&')
|
247
|
+
end
|
248
|
+
|
249
|
+
_url = "http://#{address}:#{port}" + URI.escape(_url)
|
250
|
+
return Net::HTTP.get(URI.parse(_url))
|
251
|
+
end
|
252
|
+
|
253
|
+
def inspect # :nodoc:
|
254
|
+
"#<xtractr #{address}:#{port}>"
|
255
|
+
end
|
256
|
+
end
|
257
|
+
end # Mu
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dorothy2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 23
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
|
+
- 1
|
7
8
|
- 0
|
8
9
|
- 0
|
9
|
-
|
10
|
-
version: 0.0.3
|
10
|
+
version: 1.0.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- marco riccardi
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2013-06-
|
18
|
+
date: 2013-06-07 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: net-scp
|
@@ -245,6 +245,7 @@ files:
|
|
245
245
|
- LICENSE
|
246
246
|
- README.md
|
247
247
|
- Rakefile
|
248
|
+
- TODO
|
248
249
|
- bin/dorothy_start
|
249
250
|
- bin/dorothy_stop
|
250
251
|
- bin/dparser_start
|
@@ -268,6 +269,33 @@ files:
|
|
268
269
|
- lib/dorothy2/environment.rb
|
269
270
|
- lib/dorothy2/version.rb
|
270
271
|
- lib/dorothy2/vtotal.rb
|
272
|
+
- lib/mu/xtractr.rb
|
273
|
+
- lib/mu/xtractr/about.rb
|
274
|
+
- lib/mu/xtractr/content.rb
|
275
|
+
- lib/mu/xtractr/field.rb
|
276
|
+
- lib/mu/xtractr/flow.rb
|
277
|
+
- lib/mu/xtractr/flows.rb
|
278
|
+
- lib/mu/xtractr/host.rb
|
279
|
+
- lib/mu/xtractr/packet.rb
|
280
|
+
- lib/mu/xtractr/packets.rb
|
281
|
+
- lib/mu/xtractr/service.rb
|
282
|
+
- lib/mu/xtractr/stream.rb
|
283
|
+
- lib/mu/xtractr/stream/http.rb
|
284
|
+
- lib/mu/xtractr/term.rb
|
285
|
+
- lib/mu/xtractr/test/stream/tc_http.rb
|
286
|
+
- lib/mu/xtractr/test/tc_field.rb
|
287
|
+
- lib/mu/xtractr/test/tc_flow.rb
|
288
|
+
- lib/mu/xtractr/test/tc_flows.rb
|
289
|
+
- lib/mu/xtractr/test/tc_host.rb
|
290
|
+
- lib/mu/xtractr/test/tc_packet.rb
|
291
|
+
- lib/mu/xtractr/test/tc_packets.rb
|
292
|
+
- lib/mu/xtractr/test/tc_service.rb
|
293
|
+
- lib/mu/xtractr/test/tc_stream.rb
|
294
|
+
- lib/mu/xtractr/test/tc_term.rb
|
295
|
+
- lib/mu/xtractr/test/tc_views.rb
|
296
|
+
- lib/mu/xtractr/test/tc_xtractr.rb
|
297
|
+
- lib/mu/xtractr/test/test.rb
|
298
|
+
- lib/mu/xtractr/views.rb
|
271
299
|
- share/img/Dorothy-Basic.pdf
|
272
300
|
- share/img/Setup-Advanced.pdf
|
273
301
|
- share/img/The_big_picture.pdf
|