dorothy2 0.0.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/mu/xtractr.rb ADDED
@@ -0,0 +1,257 @@
1
+ # "THE BEER-WARE LICENSE" (Revision 42):
2
+ # Mu[http://www.mudynamics.com] wrote this file. As long as you retain this
3
+ # notice you can do whatever you want with this stuff. If we meet some day,
4
+ # and you think this stuff is worth it, you can buy us a beer in return.
5
+ #
6
+ # All about pcapr
7
+ # * http://www.pcapr.net
8
+ # * http://groups.google.com/group/pcapr-forum
9
+ # * http://twitter.com/pcapr
10
+ #
11
+ # Mu Dynamics
12
+ # * http://www.mudynamics.com
13
+ # * http://labs.mudynamics.com
14
+
15
+ require 'mu/xtractr/about'
16
+ require 'mu/xtractr/content'
17
+ require 'mu/xtractr/field'
18
+ require 'mu/xtractr/flow'
19
+ require 'mu/xtractr/flows'
20
+ require 'mu/xtractr/host'
21
+ require 'mu/xtractr/packet'
22
+ require 'mu/xtractr/packets'
23
+ require 'mu/xtractr/service'
24
+ require 'mu/xtractr/stream'
25
+ require 'mu/xtractr/term'
26
+ require 'mu/xtractr/views'
27
+
28
+ module Mu # :nodoc:
29
+ # = http://www.pcapr.net/static/image/favicon.png Mu::Xtractr
30
+ # :main: Mu::Xtractr
31
+ #
32
+ # This gem is Ruby front-end to the RESTful API that <b>xtractr[http://www.pcapr.net/xtractr]</b>
33
+ # provides. We primarily use this for unit testing xtractr's API, but on
34
+ # its own this gem provides for a powerful programmable interface into
35
+ # xtractr and is a super fast way to extract information out of large pcaps.
36
+ #
37
+ # = Getting Started
38
+ # First download <b>xtractr</b> from http://www.pcapr.net/xtractr. Follow the
39
+ # instructions to index your pcap. Finally run xtractr in browse mode and then
40
+ # you can hang out in IRB poking around flows and packets.
41
+ #
42
+ # = Examples
43
+ # You can run the xtractr-gem from within IRB which makes it a fun interactive
44
+ # network forensics tool. Make sure you are running the xtractr binary in
45
+ # browse mode. Turning on auto-completion in IRB also makes it easier to try out
46
+ # different things and interactively experiment with the API:
47
+ #
48
+ # $ irb -rirb/completion -rmu/xtractr
49
+ #
50
+ # All of the examples below work off the test/test.pcap bundled with the gem.
51
+ # We'll also assume that you've done this at the start of the IRB session:
52
+ #
53
+ # irb> xtractr = Mu::Xtractr.new
54
+ #
55
+ # <b>Top DNS query names</b>
56
+ #
57
+ # We first pull out all DNS flows and then map/reduce the unique values of
58
+ # the <em>dns.qry.name</em>.
59
+ # irb> xtractr.flows('flow.service:DNS').values('dns.qry.name')
60
+ #
61
+ # <b>Services used by the top talker (based on bytes sent/received)</b>
62
+ #
63
+ # We first sum the total number of bytes using the src address as the key. The
64
+ # sum function returns the matches sorted by the #bytes. We then use the first
65
+ # object (the top talker) to in turn map/reduce the unique list of services
66
+ # supported by it.
67
+ # irb> xtractr.flows.sum('flow.src', 'flow.bytes').first.count('flow.service')
68
+ #
69
+ # <b>Generating #new pcaps based on search criteria</b>
70
+ #
71
+ # We first get a list of the unique HTTP methods in the index and then for each of
72
+ # methods, query for all the packets and then save them into a new pcap.
73
+ # irb> xtractr.packets.count('http.request.method').each { |c| c.packets.save("#{c.value}.pcap") }
74
+ #
75
+ #--
76
+ # rdoc --exclude test --force-update --inline-source --op mu/xtractr/doc --main Mu::Xtractr
77
+ class Xtractr
78
+ # Return the IP address of the xtractr instance
79
+ attr_reader :address
80
+
81
+ # Return the listening port of the xtractr instance
82
+ attr_reader :port
83
+
84
+ # Relative URL
85
+ attr_reader :relurl
86
+
87
+ # Create a new instance to connect to the xtractr binary using a
88
+ # url.
89
+ # Xtractr.create 'http://some.host:8080/'
90
+ def self.create url
91
+ uri = URI.parse url
92
+ self.new uri.host, uri.port, uri.path
93
+ end
94
+
95
+ # Create a new instance to connect to the xtractr binary running in
96
+ # browse mode.
97
+ # Xtractr.new
98
+ # Xtractr.new 'localhost', 8080
99
+ def initialize address='localhost', port=8080, relurl=nil
100
+ @address = address
101
+ @port = port
102
+ @relurl = relurl || '/'
103
+ @relurl << '/' if @relurl[-1,1] != '/'
104
+ #unless about.version =~ /^4\.5\.(svn|41604)$/
105
+ # puts "boh"
106
+ # puts "xtractr version #{about.version} out of date!"
107
+ # puts "please download a new one from http://www.pcapr.net/xtractr"
108
+ # raise
109
+ #end
110
+ end
111
+
112
+ # Fetch the meta data about the index. This includes information about
113
+ # the total number of packets, flows as well as the duration of the entire
114
+ # set of pcaps in the index.
115
+ # xtractr.about
116
+ def about
117
+ @about ||= About.new json('api/about')
118
+ end
119
+
120
+ # Fetch the list of fields in the index. The fields are only available
121
+ # if the <em>--mode forensics</em> was used during the indexing process.
122
+ # xtractr.fields
123
+ # xtractr.fields /^http/
124
+ # xtractr.fields 'http.server'
125
+ def fields regex=nil
126
+ regex = Regexp.new(regex, Regexp::IGNORECASE) if regex.is_a? String
127
+ result = (@fields ||= json 'api/fields')
128
+ result = result.select { |name| name =~ regex } if regex
129
+ return result.map { |name| Field.new self, name }
130
+ end
131
+
132
+ # Fetch a field of the given name.
133
+ # xtractr.field 'http.server'
134
+ def field name
135
+ obj = fields.find { |f| f.name == name }
136
+ raise ArgumentError, "Unknown field #{name}" if not obj
137
+ return obj
138
+ end
139
+
140
+ # Fetch the list of hosts in the index. The optional regex (or String)
141
+ # can be used to filter the hosts list.
142
+ # xtractr.hosts
143
+ # xtractr.hosts /192.168/
144
+ # xtractr.hosts '10.10'
145
+ def hosts regex=nil
146
+ regex = Regexp.new(regex, Regexp::IGNORECASE) if regex.is_a? String
147
+ result = (@hosts ||= json 'api/hosts')
148
+ rows = result['rows']
149
+ rows = rows.select { |row| row['name'] =~ regex } if regex
150
+ return rows.map { |row| Host.new self, row['name'] }
151
+ end
152
+
153
+ # Fetch a host of the given address.
154
+ # xtractr.host '192.168.1.1'
155
+ def host address
156
+ obj = hosts.find { |h| h.address == address }
157
+ raise ArgumentError, "Unknown host #{address}" if not obj
158
+ return obj
159
+ end
160
+
161
+ # Fetch the list of services in the index. The optional regex (or String)
162
+ # can be used to filter the services lists.
163
+ # xtractr.services
164
+ # xtractr.services /http/
165
+ # xtractr.services 'sip'
166
+ def services regex=nil
167
+ regex = Regexp.new(regex, Regexp::IGNORECASE) if regex.is_a? String
168
+ result = (@services ||= json 'api/services')
169
+ rows = result['rows']
170
+ rows = rows.select { |row| row['name'] =~ regex } if regex
171
+ return rows.map { |row| Service.new self, row['name'] }
172
+ end
173
+
174
+ # Fetch a service of the given name.
175
+ # xtractr.service 'dns'
176
+ def service name
177
+ obj = services.find { |s| s.name.downcase == name.downcase }
178
+ raise ArgumentError, "Unknown service #{name}" if not obj
179
+ return obj
180
+ end
181
+
182
+ # Return an iterator that can yield each flow that matched the query. If
183
+ # <em>q</em> is a Range, then it's used to extract the set of flows that match
184
+ # all of those ids.
185
+ # xtractr.flows.each { |flow| ... }
186
+ # xtractr.flows(1..10).each { |flow| ... }
187
+ # xtractr.flows("flow.src:192.168.1.1").each { |flow| ... }
188
+ def flows(q='*') # :yields: flow
189
+ if q.is_a? Range
190
+ first = q.first
191
+ last = q.last
192
+ last -= 1 if q.exclude_end?
193
+ q = "flow.id:[#{first} #{last}]"
194
+ end
195
+ return Flows.new(self, :q => q)
196
+ end
197
+
198
+ # Return the id'th flow from the index.
199
+ # xtractr.flow 1
200
+ def flow id
201
+ result = json "api/flows", :start => id, :limit => id
202
+ rows = result['rows']
203
+ raise ArgumentError, "Unknown flow #{id}" if rows.empty?
204
+ return Flow.new(self, rows[0])
205
+ end
206
+
207
+ # Return an iterator that can yield each packet that matched the query. If
208
+ # <em>q</em> is a Range, then it's used to extract the set of packets that match
209
+ # all of those ids.
210
+ # xtractr.packets.each { |pkt| ... }
211
+ # xtractr.packets(5..32).each { |pkt| ... }
212
+ # xtractr.packets("http.user.agent:mozilla").each { |pkt| ... }
213
+ def packets(q='*') # :yields: packet
214
+ if q.is_a? Range
215
+ first = q.first
216
+ last = q.last
217
+ last -= 1 if q.exclude_end?
218
+ q = "pkt.id:[#{first} #{last}]"
219
+ end
220
+ return Packets.new(self, :q => q)
221
+ end
222
+
223
+ # Return the id'th packet from the index.
224
+ # xtractr.packet 1
225
+ def packet id
226
+ result = json "api/packets", :start => id, :limit => id
227
+ rows = result['rows']
228
+ raise ArgumentError, "Unknown packet #{id}" if rows.empty?
229
+ return Packet.new(self, rows[0])
230
+ end
231
+
232
+ # Fetch the URL with the GET parameters and interpret the response body
233
+ # as a JSON object
234
+ def json url, opts={} # :nodoc:
235
+ res = get url, opts
236
+ js = JSON.parse(res)
237
+ raise ArgumentError, js['reason'] if js.is_a?(Hash) and js['error']
238
+ return js
239
+ end
240
+
241
+ # Fetch the URL and return the response body, as is
242
+ def get url, opts={} # :nodoc:
243
+ _url = relurl + url
244
+ if opts.size
245
+ _url << '?'
246
+ _url << opts.keys.map { |key| key.to_s + '=' + opts[key].to_s }.join('&')
247
+ end
248
+
249
+ _url = "http://#{address}:#{port}" + URI.escape(_url)
250
+ return Net::HTTP.get(URI.parse(_url))
251
+ end
252
+
253
+ def inspect # :nodoc:
254
+ "#<xtractr #{address}:#{port}>"
255
+ end
256
+ end
257
+ end # Mu
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dorothy2
3
3
  version: !ruby/object:Gem::Version
4
- hash: 25
4
+ hash: 23
5
5
  prerelease:
6
6
  segments:
7
+ - 1
7
8
  - 0
8
9
  - 0
9
- - 3
10
- version: 0.0.3
10
+ version: 1.0.0
11
11
  platform: ruby
12
12
  authors:
13
13
  - marco riccardi
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2013-06-02 00:00:00 Z
18
+ date: 2013-06-07 00:00:00 Z
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency
21
21
  name: net-scp
@@ -245,6 +245,7 @@ files:
245
245
  - LICENSE
246
246
  - README.md
247
247
  - Rakefile
248
+ - TODO
248
249
  - bin/dorothy_start
249
250
  - bin/dorothy_stop
250
251
  - bin/dparser_start
@@ -268,6 +269,33 @@ files:
268
269
  - lib/dorothy2/environment.rb
269
270
  - lib/dorothy2/version.rb
270
271
  - lib/dorothy2/vtotal.rb
272
+ - lib/mu/xtractr.rb
273
+ - lib/mu/xtractr/about.rb
274
+ - lib/mu/xtractr/content.rb
275
+ - lib/mu/xtractr/field.rb
276
+ - lib/mu/xtractr/flow.rb
277
+ - lib/mu/xtractr/flows.rb
278
+ - lib/mu/xtractr/host.rb
279
+ - lib/mu/xtractr/packet.rb
280
+ - lib/mu/xtractr/packets.rb
281
+ - lib/mu/xtractr/service.rb
282
+ - lib/mu/xtractr/stream.rb
283
+ - lib/mu/xtractr/stream/http.rb
284
+ - lib/mu/xtractr/term.rb
285
+ - lib/mu/xtractr/test/stream/tc_http.rb
286
+ - lib/mu/xtractr/test/tc_field.rb
287
+ - lib/mu/xtractr/test/tc_flow.rb
288
+ - lib/mu/xtractr/test/tc_flows.rb
289
+ - lib/mu/xtractr/test/tc_host.rb
290
+ - lib/mu/xtractr/test/tc_packet.rb
291
+ - lib/mu/xtractr/test/tc_packets.rb
292
+ - lib/mu/xtractr/test/tc_service.rb
293
+ - lib/mu/xtractr/test/tc_stream.rb
294
+ - lib/mu/xtractr/test/tc_term.rb
295
+ - lib/mu/xtractr/test/tc_views.rb
296
+ - lib/mu/xtractr/test/tc_xtractr.rb
297
+ - lib/mu/xtractr/test/test.rb
298
+ - lib/mu/xtractr/views.rb
271
299
  - share/img/Dorothy-Basic.pdf
272
300
  - share/img/Setup-Advanced.pdf
273
301
  - share/img/The_big_picture.pdf