flowtag 2.1.0

data.tar.gz.sig

@@ -0,0 +1,2 @@
+�b:�rT���b>k�nqG��s�������ڿ�܆��dT�(��
+��&�"<�˧�tL˸;��O++�	�؎��Ii���>�aG�yQ�k�~�6�Ů�=s����ө��f!<5ɧ��n�(Zݙę���T��[��͆ME	z����I< �"��y���}��-W�+9��/)wk�j��E�ʍ�v�ԖH�w�����e[��{�a��R���|hhI+��DN3Eb�#���!�(FF�9�h>�!#3X��

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Chris Lee, PhD
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,19 @@
+= flowtag
+
+Description goes here.
+
+== Contributing to flowtag
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Chris Lee, PhD. See LICENSE.txt for
+further details.
+

data/bin/flowtag

@@ -0,0 +1,181 @@
+#! /opt/local/bin/ruby
+# DESCRIPTION: presents the user with a GUI interface to visualize and explore flows found from a given pcap file.
+# FLOWTAG - parses and visualizes pcap data
+# Copyright (C) 2007 Christopher Lee
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+unless ARGV.length > 0
+	puts "Usage: #{$0} [-o <outputdir>] <pcapfile>"
+	exit
+end
+
+require 'tk'  # this takes a long time to load
+require 'tk/labelframe'
+require 'flowtag/flowdb'
+require 'flowtag/flowcanvas'
+require 'flowtag/flowtable'
+require 'tk-double-slider'
+
+def select_cb(flows)
+	$flowtable.clear
+	$flowtable.addflows(flows.sort_by { |fl| fl[FlowDB::ST] })
+	$flowview.clear
+	$tag_entry.delete('@0','end')
+	$currflow = nil
+end
+
+def tableselect_cb(idxs, flows)
+	return if flows.length < 1
+	flow = flows[0]
+	idx = idxs[0]
+	(st,sip,dip,sp,dp,pkts,bytes) = flow.strip.split(/\s+/)
+	$curridx = idx
+	$currflow = [sip,dip,sp,dp]
+	$flowcanvas.select_flow(sip, dip, sp.to_i, dp.to_i)
+	payload = $fdb.getflowdata(sip, dip, sp.to_i, dp.to_i, 5000)
+	# replace the gremlins that destroy the performance of the flow vie
+	payload = payload.gsub(/[\x00-\x09\x0b-\x1f\x7f-\xff]/) do |c| sprintf("\\x%02x", c[0]) end
+	$flowview.clear
+	$flowview.insert('end',payload) if payload
+	tags = $fdb.getflowtags($currflow)
+	$tag_entry.delete('@0','end')
+	$tag_entry.insert('end',tags.join(" "))
+end
+
+def tag_cb(tags)
+	return unless $currflow
+	sip,dip,sp,dp = $currflow
+	$fdb.tag_flow($currflow,tags.split(/\s+/))
+	$flowtable.update_flow($curridx, $fdb.flows[$currflow.join("|")])
+	alltags = $fdb.tags
+	$tags_list.clear
+	alltags.sort.each do |tag|
+		$tags_list.insert 'end',tag
+	end
+end
+
+def tag_select
+	tags = []
+	indices = $tags_list.curselection
+	indices.each do |i|
+		tags.push($tags_list.get(i))
+	end
+	$tags_list.focus 
+	flows = []
+	tags.each do |tag|
+		flows += $fdb.flows_taggedwith(tag)
+	end
+	select_cb(flows)
+end
+
+def finish
+	$fdb.writetagdb
+	$fdb.close
+	puts "Thank you for playing.  Please send suggestions to scholar.freenode@gmail.com"
+	exit
+end
+if ARGV[0] == '-o'
+	outputdir = ARGV[1]
+	$fdb = FlowDB.new(ARGV[2],outputdir)
+else
+	$fdb = FlowDB.new(ARGV[0])
+end
+pkt_min = byte_min = time_min = 2**32
+pkt_max = byte_max = time_max = 0
+$fdb.flows.each do |key,flow|
+	pkt_min = flow[FlowDB::PKTS] if flow[FlowDB::PKTS] < pkt_min
+	pkt_max = flow[FlowDB::PKTS] if flow[FlowDB::PKTS] > pkt_max
+	byte_min = flow[FlowDB::BYTES] if flow[FlowDB::BYTES] < byte_min
+	byte_max = flow[FlowDB::BYTES] if flow[FlowDB::BYTES] > byte_max
+	time_min = flow[FlowDB::ST] if flow[FlowDB::ST] < time_min
+	time_max = flow[FlowDB::ST] if flow[FlowDB::ST] > time_max
+end
+root = TkRoot.new() {
+	title "FlowTag v2.0"
+	protocol('WM_DELETE_WINDOW', proc{ finish })
+}
+root.bind('Control-q', proc{ finish })
+root.bind('Control-c', proc{ finish })
+trap('SIGINT') { finish }
+TkPalette.setPalette('background','grey20','foreground','white','disabledBackground','grey20','disabledForeground','grey30')
+left_frame = TkFrame.new(root)
+left_top_frame = TkLabelFrame.new(left_frame,:text=>'Flow Table')
+left_mid_frame = TkLabelFrame.new(left_frame,:text=>'Flow Tags')
+left_bot_frame = TkLabelFrame.new(left_frame,:text=>'Payload View')
+right_frame = TkFrame.new(root)
+right_top_frame = TkLabelFrame.new(right_frame,:text=>'Connection Visualization')
+right_bot_frame = TkLabelFrame.new(right_frame,:text=>'Filters')
+tags_frame = TkLabelFrame.new(root, :text=>'Tags List')
+
+left_frame.grid(:row=>0,:column=>0,:sticky=>'new')
+left_top_frame.grid(:row=>0,:column=>0,:sticky=>'news')
+left_mid_frame.grid(:row=>1,:column=>0,:sticky=>'news')
+left_bot_frame.grid(:row=>2,:column=>0,:sticky=>'news')
+right_frame.grid(:row=>0,:column=>1,:sticky=>'new')
+right_top_frame.grid(:row=>0,:column=>0,:sticky=>'news')
+right_bot_frame.grid(:row=>1,:column=>0,:sticky=>'news')
+tags_frame.grid(:row=>0,:column=>2,:sticky=>'new')
+
+# LEFT FRAME
+$flowtable = FlowTable.new(left_top_frame,$fdb.flows.values.sort_by { |fl| fl[FlowDB::ST] })
+$flowtable.pack(:side=>'top',:expand=>1,:fill=>'both',:anchor=>'n')
+$flowtable.set_select_cb(proc { |idxs,flows| tableselect_cb(idxs,flows) })
+
+$tag_entry = TkEntry.new(left_mid_frame) {
+	font TkFont.new('Monaco 12')
+}
+$tag_entry.bind('Return', proc { |e| tag_cb($tag_entry.get) } )
+$tag_entry.pack(:side=>'right', :expand=>1, :fill=>'x')
+
+flowview_scrollbar = TkScrollbar.new(left_bot_frame)
+$flowview = flowview = TkText.new(left_bot_frame) {
+	font TkFont.new('Monaco 12')
+	wrap 'char'
+	height 30
+	width 60
+}
+flowview_scrollbar.pack(:side=>'right', :fill=>'y')
+$flowview.pack(:side=>'bottom',:expand=>1,:fill=>'both')
+$flowview.yscrollbar(flowview_scrollbar);
+
+# TAGS FRAME
+$tags_list = TkScrollbox.new(tags_frame) {
+	font TkFont.new('Monaco 12')
+	height 40
+	width 20
+}
+$tags_list.pack(:fill=>'y',:expand=>1)
+$fdb.tags.sort.each do |tag|
+	$tags_list.insert('end',tag)
+end
+$tags_list.bind('<ListboxSelect>',proc { tag_select })
+
+# RIGHT FRAME
+$flowcanvas = FlowCanvas.new(right_top_frame,$fdb.flows.sort_by { |k,fl| fl[FlowDB::ST] })
+$timeslide = TkDoubleSlider.new(right_bot_frame, :min=>time_min, :max=>time_max, :low=>time_min, :high=>time_max, :snap => 300, :label=>'Time', :valuefmt => proc { |x| Time.at(x).strftime("%H:%M") }, :deltafmt => proc { |x| sprintf("%0.2f hours", (x/3600.0)) })
+$pktslide = TkDoubleSlider.new(right_bot_frame, :min=>1, :max=>pkt_max, :low=>1, :high=>pkt_max, :logbase => true, :snap => 1.0, :label=>'Packets')
+$byteslide = TkDoubleSlider.new(right_bot_frame, :min=>1, :max=>byte_max, :low=>1, :high=>byte_max, :logbase => true, :snap => 1.0, :label=>'Bytes')
+$flowcanvas.pack
+$timeslide.pack(:side=>'top')
+$pktslide.pack
+$byteslide.pack(:side=>'bottom')
+$flowcanvas.set_select_cb(proc { |x| select_cb(x) })
+$timeslide.change_cb = proc { |low,high| $flowcanvas.set_time_range(low,high) };
+$pktslide.change_cb = proc { |low,high| $flowcanvas.set_packet_range(low,high) };
+$byteslide.change_cb = proc { |low,high| $flowcanvas.set_byte_range(low,high) };
+$currflow = nil
+
+Tk.mainloop()
+

data/bin/ftlistflows

@@ -0,0 +1,29 @@
+#! /opt/local/bin/ruby
+# DESCRIPTION: is part of the flowtag toolkit and prints the flows from a flowdb
+# FLOWTAG - parses and visualizes pcap data
+# Copyright (C) 2007 Christopher Lee
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'flowtag/flowdb'
+
+unless ARGV.length > 0
+	puts "Usage: $0 <pcapfile>"
+	exit
+end
+ARGV.each do |f|
+	fdb = FlowDB.new(f)
+	fdb.dumpflows
+end
+

data/bin/ftpcap2flowdb

@@ -0,0 +1,26 @@
+#! /opt/local/bin/ruby
+# DESCRIPTION: is part of the flowtag toolkit and generates a flow database from a pcap file
+# Copyright (C) 2007 Christopher Lee
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'flowtag/flowdb'
+
+unless ARGV.length > 0
+	puts "Usage: $0 <pcapfile>"
+	exit
+end
+ARGV.each do |f|
+	FlowDB.new(f)
+end

data/bin/ftprintflow

@@ -0,0 +1,27 @@
+#! /opt/local/bin/ruby
+# DESCRIPTION: is part of the flowtag toolkit and prints the contents of an indicated flow
+# FLOWTAG - parses and visualizes pcap data
+# Copyright (C) 2007 Christopher Lee
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'flowtag/flowdb'
+
+unless ARGV.length == 5
+	puts "Usage: $0 <pcapfile> <sip> <dip> <sp> <dp>"
+	exit
+end
+(f,sip,dip,sp,dp) = ARGV
+fdb = FlowDB.new(f)
+puts fdb.getflowdata(sip, dip, sp.to_i, dp.to_i)

data/lib/flowtag.rb

@@ -0,0 +1,4 @@
+require 'flowtag/flowcanvas'
+require 'flowtag/flowdb'
+require 'flowtag/flowtable'
+require 'flowtag/pcapparser'

data/lib/flowtag/flowcanvas.rb

@@ -0,0 +1,124 @@
+# DESCRIPTION: is part of the flowtag toolkit and provides a Tk widget for visualizing and selecting flows on a canvas
+# FLOWTAG - parses and visualizes pcap data
+# Copyright (C) 2007 Christopher Lee
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'date'
+require 'tk-parallel-coordinates'
+
+module FlowTag
+	class FlowCanvas
+		def cb_select(tuples)
+			@selected_flows = []
+			tuples.each do |t|
+				key = t.join("|")
+				@selected_flows += @flow_keys[key]
+			end
+			@select_cb.call @selected_flows if @select_cb
+		end
+
+		def select_flow(sip,dip,sp,dp)
+			@pcp.set_tuple_state(@cflow,TK::ParallelCoordinates::STATE_NORMAL) if @cflow
+			key = dp.to_s+"|"+sip
+			@cflow = key
+			@pcp.set_tuple_state(key,TK::ParallelCoordinates::STATE_CURRENT)
+		end
+
+		def set_select_cb(callback)
+			@select_cb = callback
+		end
+
+		def set_time_range(low, high)
+			@time_low = low
+			@time_high = high
+			filter
+		end
+
+		def set_byte_range(low, high)
+			@byte_low = low
+			@byte_high = high
+			filter
+		end
+
+		def set_packet_range(low, high)
+			@pkt_low = low
+			@pkt_high = high
+			filter
+		end
+
+		def filter
+			@flow_keys.each do |key,flows|
+				flows.each do |fl|
+					if fl[FlowDB::PKTS] < @pkt_low or fl[FlowDB::PKTS] > @pkt_high or 
+						fl[FlowDB::BYTES] < @byte_low or fl[FlowDB::BYTES] > @byte_high or 
+						fl[FlowDB::ST] < @time_low or fl[FlowDB::ST] > @time_high
+						@pcp.set_tuple_state(key,TK::ParallelCoordinates::STATE_FILTERED)
+					else
+						@pcp.set_tuple_state(key,TK::ParallelCoordinates::STATE_NORMAL)
+					end
+				end
+			end
+		end
+
+		def pack(*args)
+			@pcp.pack(*args)
+		end
+
+		def initialize(parent, flows)
+			hostseen = {}
+			hosts = []
+			@selected_flows = @flows = flows
+			flows.each do |k,fl|
+				sip = fl[FlowDB::SIP]
+				next if hostseen[sip]
+				hosts.push(sip)
+				hostseen[sip]=1
+			end
+			model = [ 
+				{ 
+					:name => 'Port',
+					:type => 'range',
+					:scale => '3rt',
+					:min => 0,
+					:max => 65535,
+					:ofmt => '%d',
+					#:items => [1,22,80,137,443,1024,5900,6667,31337,65335]
+				},
+				{
+					:name => 'Host',
+					:type => 'list',
+					:list => hosts
+				}
+			]
+			@pcp = TK::ParallelCoordinates.new(parent, 500, 360, model)
+			@pcp.set_select_cb( proc { |tuples| cb_select(tuples) } )
+			@flow_keys = {}
+			@pkt_low = @byte_low = @pkt_high = @byte_high = @time_high = 0
+			@time_low = 2**32
+			flows.each do |k,fl|
+				key = fl[FlowDB::DP].to_s+"|"+fl[FlowDB::SIP]
+				skip = (@flow_keys[key]) ? true:false
+				@flow_keys[key] = [] unless @flow_keys[key]
+				@flow_keys[key].push(fl)
+				@pkt_high = fl[FlowDB::PKTS] if fl[FlowDB::PKTS] > @pkt_high
+				@byte_high = fl[FlowDB::BYTES] if fl[FlowDB::BYTES] > @byte_high
+				@time_low = fl[FlowDB::ST] if fl[FlowDB::ST] < @time_low
+				@time_high = fl[FlowDB::ST] if fl[FlowDB::ST] > @time_high
+				next if skip
+				@pcp.addtuple(key,TK::ParallelCoordinates::STATE_NORMAL,[fl[FlowDB::DP],fl[FlowDB::SIP]])
+			end
+		end
+	end
+end

data/lib/flowtag/flowdb.rb

@@ -0,0 +1,266 @@
+# DESCRIPTION: is part of the flowtag toolkit and wraps a flowdb
+# FLOWTAG - parses and visualizes pcap data
+# Copyright (C) 2007 Christopher Lee
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'flowtag/pcapparser'
+require 'ipaddr'
+
+module FlowTag
+	class FlowDB
+		ST = 0
+		SIP = 1
+		DIP = 2
+		SP = 3
+		DP = 4
+		PKTS = 5
+		BYTES = 6
+		FIRST_PKT = 7
+		TAGS = 8
+
+		attr_reader :flows, :tags_flows
+		def initialize(pcapfile,basedir=nil)
+			@pcapfile = pcapfile
+			basedir = File.dirname(pcapfile) unless basedir
+			basename = basedir+"/"+File.basename(pcapfile)
+			raise "Cannot find pcapfile, #{pcapfile}" unless File.exists?(pcapfile)
+			@pcapfh = File.new(pcapfile, 'rb')
+			unless File.exists?(basename+".flows") and File.exists?(basename+'.pkts') and File.exists?(basename+'.tags')
+				# the flow and packet database are missing, let's generate them
+				flowdb = File.new(basename+".flows", 'wb')
+				pktdb = File.new(basename+".pkts", 'wb')
+				tagdb = File.new(basename+".tags", 'w')
+				create_flowdb(basename, flowdb, pktdb)
+				flowdb.close
+				pktdb.close
+				tagdb.close
+				@flowdb = File.new(basename+".flows", 'rb')
+				@pktdb = File.new(basename+".pkts", 'rb')
+				@tagdb = File.new(basename+".tags", 'r')
+				@flows = readflows
+				@tags_flows = {}
+				@flows_tags = {}
+			else
+				@flowdb = File.new(basename+".flows", 'rb')
+				@pktdb = File.new(basename+".pkts", 'rb')
+				@tagdb = File.new(basename+".tags", 'r')
+				@flows = readflows
+				# readtags must be called AFTER readflows
+				@tags_flows, @flows_tags = readtags
+			end
+			@tags_updated = false
+		end
+
+		def close
+			@pcapfh.close if @pcapfh
+			@pktdb.close if @pktdb
+			@flowdb.close if @flowdb
+			@tagdb.close if @tagdb
+		end
+
+		def create_flowdb(pcapfile, flowdb, pktdb)
+			offset = 24 # offset into the pcap file, starts at 24 to skip header
+			pktid = 0   # database id of the pkt, increments for each matching packet
+			flows = {}  # flow hash to store and check for keys
+			pkts = []   # stores the packet database
+			pcap = PcapParser.new(File.new(pcapfile,'rb'))
+			pcap.each do |pkt|
+				unless pkt.tcp?
+					offset += 16 + pkt.length
+					next
+				end
+				tuple = [pkt.ip_src, pkt.tcp_sport, pkt.tcp_dport, pkt.ip_dst]
+				key = tuple.join "|"
+				rkey = tuple.reverse.join "|"
+				if flows[rkey]
+					key = rkey
+				end
+				if flows[key]
+					last_pkt_id = flows[key][:last_pkt]
+					pkts[last_pkt_id][:next_pkt] = pktid
+					flows[key][:last_pkt] = pktid
+				else
+					flows[key] = { :st => pkt.time, :sip => pkt.ip_src, :dip => pkt.ip_dst, :sp => pkt.tcp_sport, :dp => pkt.tcp_dport, :pkts => 0, :bytes => 0 }
+					flows[key][:first_pkt] = flows[key][:last_pkt] = pktid
+				end
+				flows[key][:pkts] += 1
+				flows[key][:bytes] += pkt.length
+				pkts[pktid] = { :offset => offset, :next_pkt => 0 }
+				offset += 16 + pkt.length
+				pktid+=1
+			end
+			pcap.close
+
+			# write out the flow database and the packet database
+			flows.sort_by{|key,flow| flow[:st]}.each do |key,flow|
+				flowdb.write( [ flow[:st], flow[:sip], flow[:dip], flow[:sp], flow[:dp], flow[:pkts], flow[:bytes], flow[:first_pkt] ].pack("NNNnnNNn") )
+				flows[key] = [flow[:st], [flow[:sip]].pack("N").unpack("C4").join("."), [flow[:dip]].pack("N").unpack("C4").join("."), flow[:sp], flow[:dp], flow[:pkts], flow[:bytes], flow[:first_pkt], []]
+			end
+			pkts.each do |pkt|
+				pktdb.write([pkt[:offset],pkt[:next_pkt]].pack("Nn"))
+			end
+			return flows
+		end
+
+		def dumpflows
+			@flows.sort_by { |k,f| f[ST].to_i }.each do |key,flow|
+				puts flow.join(" ")
+			end
+		end
+
+		def readflows
+			flows = {}
+			@flowdb.seek(0)
+			while ! @flowdb.eof?
+				(st,sip,dip,sp,dp,pkts,bytes,first_pkt) = @flowdb.read(26).unpack("NNNnnNNn")
+				sip = IPAddr.ntop([sip].pack("N"))
+				dip = IPAddr.ntop([dip].pack("N"))
+				key = [sip,dip,sp,dp].join("|")
+				flows[key] = [st,sip,dip,sp,dp,pkts,bytes,first_pkt,[]]
+			end
+			flows
+		end
+
+		def getflows
+			return @flows
+		end
+
+		def readtags
+			@tagdb.seek 0
+			flows_tags = {}
+			tags_flows = {}
+			@tagdb.each_line do |l|
+				(sip,dip,sp,dp,*tags)=l.strip.split(/\|/)
+				key = [sip,dip,sp,dp].join("|")
+				flows_tags[key] = tags
+				@flows[key][TAGS] = tags
+				tags.each do |tag|
+					tags_flows[tag] = [] unless tags_flows[tag]
+					tags_flows[tag].push(key)
+				end 
+			end
+			[tags_flows, flows_tags]
+		end
+
+		def writetagdb
+			return true unless @tags_updated
+			@tagdb.close if @tagdb
+			tagdb = File.new(@pcapfile+'.tags', 'w')
+			@flows_tags.each do |key,tags|
+				tagdb.puts key+"|"+tags.join("|")
+			end
+			tagdb.close
+			@tagdb = File.new(@pcapfile+'.tags', 'r')
+			@tags_updated = false
+			true
+		end
+
+		def tag_flow(flow, tags)
+			@tags_updated = true
+			tags.uniq!
+			key = flow.join("|")
+			if @flows_tags[key]
+				currtags = @flows_tags[key]
+				currtags.each do |tag|
+					@tags_flows[tag].delete(key)
+					@tags_flows.delete(tag) if @tags_flows[tag].length == 0
+				end
+			end
+			@flows_tags[key] = tags
+			@flows[key][TAGS] = tags
+			tags.each do |tag|
+				@tags_flows[tag] = [] unless @tags_flows[tag]
+				@tags_flows[tag].push(key)
+			end
+		end
+
+		def getflowtags(flow)
+			key = flow.join("|")
+			@flows_tags[key] || []
+		end
+
+		def flows_taggedwith(tag)
+			keys = @tags_flows[tag]
+			flows = []
+			if keys
+				keys.each do |key|
+					flows.push(@flows[key])
+				end
+			end
+			flows
+		end
+
+		def tags
+			@tags_flows.keys
+		end
+
+		def getfirstpktid(sip, dip, sp, dp)
+			@flows.each do |key,flow|
+				if flow[SIP] == sip and flow[DIP] == dip and flow[SP] == sp and flow[DP] == dp
+					return flow[FIRST_PKT]
+				end
+			end
+			-1
+		end
+
+		def getpktrec(pktid)
+			offset = 6*pktid
+			@pktdb.seek(offset)
+			(poff, pktid) = @pktdb.read(6).unpack("Nn")
+			[poff, pktid] 
+		end
+
+		def getdata(offset)
+			@pcapfh.seek(offset)
+			# this needs to be endian sensitive...
+			if @endian
+				(tv_sec, tv_usec, caplen, origlen) = @pcapfh.read(16).unpack(@endian)
+			else
+				(tv_sec, tv_usec, caplen, origlen) = @pcapfh.read(16).unpack("VVVV")
+				@endian = "VVVV"
+				if caplen > 5000
+					@pcapfh.seek(offset)
+					(tv_sec, tv_usec, caplen, origlen) = @pcapfh.read(16).unpack("NNNN")
+					@endian = "NNNN"
+				end
+			end
+			pkt = @pcapfh.read(caplen)
+			type = (pkt[12,2].unpack("n"))[0]
+			return nil unless type == 0x0800
+			return nil unless pkt[14] == 0x45
+			return nil unless pkt[14+10-1] == 0x06
+			tcp_header_len = (pkt[14+20+12]>>4)<<2
+			pkt[14+20+tcp_header_len,1000] || ''
+		end
+
+		def getflowdata_frompkt(first_pkt, limit=nil)
+			(poff,npkt) = fp_rec = getpktrec(first_pkt)
+			payload = getdata(poff)
+			while npkt != 0
+				(poff,npkt) = getpktrec(npkt)
+				data = getdata(poff)
+				payload += data if data
+				break if limit and payload.length > limit
+			end
+			payload
+		end
+
+		def getflowdata(sip, dip, sp, dp, limit=nil)
+			first_pkt = getfirstpktid(sip, dip, sp, dp)
+			return '' if first_pkt == -1
+			return getflowdata_frompkt(first_pkt, limit)
+		end
+	end
+end

data/lib/flowtag/flowtable.rb

@@ -0,0 +1,107 @@
+# DESCRIPTION: is part of the flowtag toolkit and provides a TK widget for listing the flows
+# FLOWTAG - parses and visualizes pcap data
+# Copyright (C) 2007 Christopher Lee
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'flowtag/flowdb'
+
+module FlowTag
+	class FlowTable
+		@@column_lengths = [8,15,15,5,5,6,8,20]
+		@@column_names = ['Time', 'Source IP', 'Dest. IP', 'SPort', 'DPort', 'Pkts', 'Bytes', 'Tags']
+
+		def addflow(flow)
+			entry = Time.at(flow[FlowDB::ST]).strftime("%H:%M:%S")+" "
+			(1..@@column_lengths.length-2).each do |i|
+				entry += flow[i].to_s.rjust(@@column_lengths[i])+" "
+			end
+			entry += flow[FlowDB::TAGS].join(" ")[0,20]
+			@scrollbox.insert 'end', entry
+		end
+
+		def addflows(flows)
+			flows.each do |flow|
+				addflow(flow)
+			end
+		end
+
+		def update_flow(idx, flow)
+			entry = Time.at(flow[FlowDB::ST]).strftime("%H:%M:%S")+" "
+			(1..@@column_lengths.length-2).each do |i|
+				entry += flow[i].to_s.rjust(@@column_lengths[i])+" "
+			end
+			entry += flow[FlowDB::TAGS].join(" ")[0,20]
+			@scrollbox.delete idx
+			@scrollbox.insert idx, entry
+		end
+
+
+		def clear
+			@scrollbox.clear
+		end
+
+		def pack(*args)
+			@tableframe.pack(*args)
+		end
+
+		def unpack
+			@tableframe.unpack
+		end
+
+		def selected
+			items = []
+			indices = @scrollbox.curselection
+			indices.each do |i|
+				items.push(@scrollbox.get(i))
+			end
+			if @select_cb
+				@select_cb.call indices, items
+			end
+			@scrollbox.focus
+		end
+
+		def set_select_cb(callback)
+			@select_cb = callback
+		end
+
+		def initialize(parent, flows)
+			@rows = 0
+			@tableframe = TkFrame.new(parent)
+			header = ''
+			(0..@@column_names.length-1).each do |i|
+				header +=  @@column_names[i].center(@@column_lengths[i])+"|"
+			end
+			@table_header = TkLabel.new(@tableframe) {
+				text header
+				font TkFont.new('Monaco 12 bold')
+				anchor 'sw'
+				height 1
+				padx 0
+				pady 0
+				foreground 'lightblue'
+			}
+			@scrollbox = scrollbox = TkScrollbox.new(@tableframe) {
+				setgrid 'yes'
+				takefocus 'yes'
+				width 59
+				font TkFont.new('Monaco 12')
+			}
+			@table_header.pack(:side=>'top',:fill=>'x')
+			scrollbox.pack(:side=>'top',:fill=>'both',:expand=>1)
+			scrollbox.bind('<ListboxSelect>', proc { |x| selected() })
+			addflows(flows)
+		end
+	end
+end

data/lib/flowtag/pcapparser.rb

@@ -0,0 +1,107 @@
+# DESCRIPTION: is part of the flowtag toolkit and simply parses a pcap file.
+# Copyright (C) 2007 Christopher Lee
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Lesser GNU General Public License for more details.
+# 
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+module FlowTag
+	class PcapParser
+		LINKTYPE_ETH = 0x0001
+		LINKTYPE_SLL = 0x0071
+
+		def initialize(pcapfh)
+			@offset = 0
+			@bigendian = nil
+			@fh = pcapfh
+			@fh.seek 0
+			magic = @fh.read(4).unpack("N")[0]
+			@bigendian = (magic == 0xa1b2c3d4) ? true : false
+			endian = (@bigendian) ? "nnNNNN" : "vvVVVV"
+			@version_major, @version_minor, @zone, @significant_figures, @snaplength, @linktype = @fh.read(20).unpack(endian)
+			@offset += 24
+			if @linktype != LINKTYPE_ETH
+				puts "Only ethernet is supported, sorry."
+				exit
+			end
+		end
+
+		def nextpkt
+			endian = (@bigendian) ? "NNNN" : "VVVV"
+			pkt = {}
+			tv_sec, tv_usec, caplen, origlen = @fh.read(16).unpack(endian)
+			time = tv_sec + (tv_usec / 1E6)
+			data = @fh.read(caplen)
+			@offset += 16+caplen
+			return Packet.new(time, data)
+		end
+
+		def each
+			while ! @fh.eof?
+				yield nextpkt
+			end
+		end
+
+		def close
+			@fh.close unless @fh.tty?
+		end
+	end
+
+	class Packet
+		attr_reader :time, :data, :ip_src, :ip_dst, :sport, :dport, :tcp_sport, :tcp_dport, :udp_sport, :udp_dport, :length
+		def initialize(time, data)
+			@time = time
+			@data = data
+			@length = data.length
+			@ip = @tcp = @udp = false
+			@ip_src = @ip_dst = @sport = @dport = @tcp_sport = @tcp_dport = @udp_sport = @udp_dport = nil
+			@ip = (data[12,2].unpack("n")[0] == 0x0800) ? true : false
+			offset = 14
+			if @ip
+				@ip_hlen = (data[offset].ord & 0x0f) << 2
+				@ip_proto = data[offset+9].ord
+				@ip_src, @ip_dst = data[offset+12,8].unpack("NN")
+				offset += @ip_hlen
+				@tcp = true if @ip_proto == 0x06
+				@udp = true if @ip_proto == 0x11
+				if @tcp
+					@sport, @dport = data[offset,4].unpack("nn")
+					@tcp_sport = @sport
+					@tcp_dport = @dport
+					@tcp_hlen = (data[offset+12]>>4)<<2
+					offset += @tcp_hlen
+				elsif @udp
+					@sport, @dport = data[offset,4].unpack("nn")
+					@udp_sport = @sport
+					@udp_dport = @dport
+					offset += 8
+				end
+			end
+			@data_offset = offset
+		end
+
+		def ip?
+			@ip
+		end
+
+		def udp?
+			@udp
+		end
+
+		def tcp?
+			@tcp
+		end
+
+		def payload
+			@data[@data_offset,10000]
+		end
+	end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'flowtag'
+
+class Test::Unit::TestCase
+end

data/test/test_flowtag.rb

@@ -0,0 +1,63 @@
+require 'helper'
+require 'pp'
+class TestFlowtag < Test::Unit::TestCase
+	flow = ['192.168.44.100', '72.14.207.99', 50697, 80]
+	
+	should "create a flowdb from the test.pcap and dump the flows" do
+		File.unlink('test/test.pcap.flows') if File.exists?('test/test.pcap.flows')
+		File.unlink('test/test.pcap.pkts') if File.exists?('test/test.pcap.pkts')
+		File.unlink('test/test.pcap.tags') if File.exists?('test/test.pcap.tags')
+		fdb = FlowTag::FlowDB.new('test/test.pcap')
+		assert(File.exists?('test/test.pcap.flows'))
+		assert(File.exists?('test/test.pcap.pkts'))
+		assert(File.exists?('test/test.pcap.tags'))
+		fdb.dumpflows
+	end
+	
+	should "get the first pktid of the test.pcap" do
+		fdb = FlowTag::FlowDB.new('test/test.pcap')
+		pid = fdb.getfirstpktid(*flow)
+		assert_equal(0,pid)
+	end
+	
+	should "return no tags for the first flow" do
+		fdb = FlowTag::FlowDB.new('test/test.pcap')
+		tags = fdb.getflowtags(flow)
+		assert_equal(0,tags.length)
+	end
+	
+	should "get all flows tagged with test should be empty" do
+		fdb = FlowTag::FlowDB.new('test/test.pcap')
+		flows = fdb.flows_taggedwith("test")
+		assert_equal(0,flows.length)
+	end
+	
+	should "tag the first flow with test and retrieve it" do
+		fdb = FlowTag::FlowDB.new('test/test.pcap')
+		fdb.tag_flow(flow,["test"])
+		flows = fdb.flows_taggedwith("test")
+		assert_equal(1,flows.length)
+	end
+	
+	should "write the tags database and reload" do
+		fdb = FlowTag::FlowDB.new('test/test.pcap')
+		fdb.tag_flow(flow,["test"])
+		fdb.writetagdb
+		fdb = FlowTag::FlowDB.new('test/test.pcap')
+		flows = fdb.flows_taggedwith("test")
+		assert_equal(1, flows.length)
+		fdb.tag_flow(flow,[])
+		fdb.writetagdb
+		fdb = FlowTag::FlowDB.new('test/test.pcap')
+		flows = fdb.flows_taggedwith("test")
+		assert_equal(0, flows.length)
+	end
+	
+	should "list all the tags and receive one, test" do
+		fdb = FlowTag::FlowDB.new('test/test.pcap')
+		fdb.tag_flow(flow,["test"])
+		tags = fdb.tags
+		assert_equal(1, tags.length)
+		assert_equal("test", tags[0])
+	end
+end

metadata

@@ -0,0 +1,229 @@
+--- !ruby/object:Gem::Specification 
+name: flowtag
+version: !ruby/object:Gem::Version 
+  hash: 11
+  prerelease: 
+  segments: 
+  - 2
+  - 1
+  - 0
+  version: 2.1.0
+platform: ruby
+authors: 
+- Chris Lee
+autorequire: 
+bindir: bin
+cert_chain: 
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIDYjCCAkqgAwIBAgIBADANBgkqhkiG9w0BAQUFADBXMREwDwYDVQQDDAhydWJ5
+  Z2VtczEYMBYGCgmSJomT8ixkARkWCGNocmlzbGVlMRMwEQYKCZImiZPyLGQBGRYD
+  ZGhzMRMwEQYKCZImiZPyLGQBGRYDb3JnMB4XDTExMDIyNzE1MzAxOVoXDTEyMDIy
+  NzE1MzAxOVowVzERMA8GA1UEAwwIcnVieWdlbXMxGDAWBgoJkiaJk/IsZAEZFghj
+  aHJpc2xlZTETMBEGCgmSJomT8ixkARkWA2RoczETMBEGCgmSJomT8ixkARkWA29y
+  ZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALNM1Hjs6q58sf7Jp64A
+  vEY2cnRWDdFpD8UWpwaJK5kgSHOVgs+0mtszn+YlYjmx8kpmuYpyU4g9mNMImMQe
+  ow8pVsL4QBBK/1Ozgdxrsptk3IiTozMYA+g2I/+WvZSEDu9uHkKe8pvMBEMrg7RJ
+  IN7+jWaPnSzg3DbFwxwOdi+QRw33DjK7oFWcOaaBqWTUpI4epdi/c/FE1I6UWULJ
+  ZF/Uso0Sc2Pp/YuVhuMHGrUbn7zrWWo76nnK4DTLfXFDbZF5lIXT1w6BtIiN6Ho9
+  Rdr/W6663hYUo3WMsUSa3I5+PJXEBKmGHIZ2TNFnoFIRHha2fmm1HC9+BTaKwcO9
+  PLcCAwEAAaM5MDcwCQYDVR0TBAIwADAdBgNVHQ4EFgQURzsNkZo2rv86Ftc+hVww
+  RNICMrwwCwYDVR0PBAQDAgSwMA0GCSqGSIb3DQEBBQUAA4IBAQBRRw/iNA/PdnvW
+  OBoNCSr/IiHOGZqMHgPJwyWs68FhThnLc2EyIkuLTQf98ms1/D3p0XX9JsxazvKT
+  W/in8Mm/R2fkVziSdzqChtw/4Z4bW3c+RF7TgX6SP5cKxNAfKmAPuItcs2Y+7bdS
+  hr/FktVtT2iAmISRnlEbdaTpfl6N2ZWNT83khV6iOs5xRkX/+0e+GgAv9mE6nqr1
+  AkuDXMhposxcnFZUrZ3UtMPEe/JnyP7Vv6pvr3qtZm8FidFZU91+rX/fwdyBU8RP
+  /5l8uLWXXNt1wEbtu4N1I66LwTK2iRrQZE8XtlgZGbxYDFUkiurq3OafF2YwRs6W
+  6yhklP75
+  -----END CERTIFICATE-----
+
+date: 2011-03-08 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 0
+        - 1
+        - 0
+        version: 0.1.0
+  requirement: *id001
+  prerelease: false
+  name: tk-double-slider
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 0
+        - 1
+        - 0
+        version: 0.1.0
+  requirement: *id002
+  prerelease: false
+  name: tk-parallel-coordinates
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id003
+  prerelease: false
+  name: shoulda
+  type: :development
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  requirement: *id004
+  prerelease: false
+  name: bundler
+  type: :development
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 1
+        - 5
+        - 2
+        version: 1.5.2
+  requirement: *id005
+  prerelease: false
+  name: jeweler
+  type: :development
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id006
+  prerelease: false
+  name: rcov
+  type: :development
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 0
+        - 1
+        - 0
+        version: 0.1.0
+  requirement: *id007
+  prerelease: false
+  name: tk-double-slider
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 0
+        - 1
+        - 0
+        version: 0.1.0
+  requirement: *id008
+  prerelease: false
+  name: tk-parallel-coordinates
+  type: :runtime
+description: presents the user with a GUI interface to visualize and explore flows found from a given pcap file
+email: rubygems@chrislee.dhs.org
+executables: 
+- flowtag
+- ftlistflows
+- ftpcap2flowdb
+- ftprintflow
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- bin/flowtag
+- bin/ftlistflows
+- bin/ftpcap2flowdb
+- bin/ftprintflow
+- lib/flowtag.rb
+- lib/flowtag/flowcanvas.rb
+- lib/flowtag/flowdb.rb
+- lib/flowtag/flowtable.rb
+- lib/flowtag/pcapparser.rb
+- LICENSE.txt
+- README.rdoc
+- test/helper.rb
+- test/test_flowtag.rb
+has_rdoc: true
+homepage: https://rubygems.org/gems/flowtag
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.1
+signing_key: 
+specification_version: 3
+summary: FlowTag visualizes pcap files for forensic analysis
+test_files: 
+- test/helper.rb
+- test/test_flowtag.rb

metadata.gz.sig

@@ -0,0 +1 @@
+'mnN�V���&���KA� �6#ָ�(�g�6��Zj"�����1�Y��zq1�"�X,k����QK�nY��u�2����Ɏ6�o��fO%Z
���ȭC��"lX�_�I���Wbn=ϤO>K�t�o�"�B�n4?��B���g��`aD�Ͽ�P#��픳�2�<䷛��L�}��͟��������O-�/]���$���74a�eCXEq8V�-�`^