DIY-pcap 0.3.5 → 0.3.6

data/lib/diy/builder.rb

@@ -34,6 +34,10 @@ module DIY
       @timeout = timeout
     end
     
+    def error_on_stop(*)
+      @error_on_stop = true
+    end
+    
     def client(ip_or_iport)
       @curi = ip_or_iport_with_default(ip_or_iport, 7878)
     end
@@ -78,6 +82,9 @@ module DIY
       if @filter
         @client.filter(@filter)
         @server.filter(@filter)
+      else
+        @client.filter("")
+        @server.filter("")
       end
     end
     
@@ -90,6 +97,7 @@ module DIY
       controller = Controller.new( @client, @server, @offline, @strategy_builder )
       controller.before_send(&@before_send_hook)
       controller.timeout(@timeout) if @timeout
+      controller.error_on_stop if @error_on_stop
       controller.run
     end
     

data/lib/diy/controller.rb

@@ -10,9 +10,11 @@ module DIY
       @strategy = strategy
       @before_send = nil
       @timeout = nil
+      @error_on_stop = nil
     end
     
     def run
+      do_trap
       client = @client
       server = @server
       
@@ -24,25 +26,40 @@ module DIY
       
       loop do
         begin
-          pkts = @offline.nexts
+          pkts, where = @offline.nexts
+          case where
+          when :A
+            client, server = @client, @server
+          when :B
+            client, server = @server, @client
+          end
           one_round( client, server, pkts )
-          client, server = server, client
         rescue HopePacketTimeoutError, UserError, FFI::PCap::LibError => e
           DIY::Logger.warn( "Timeout: Hope packet is #{pkts[0].pretty_print} ") if e.kind_of?(HopePacketTimeoutError)
           @fail_count += 1
-          begin
-            @offline.next_pcap
-            server.terminal
-          rescue EOFError
+          if @error_on_stop and e.kind_of?(HopePacketTimeoutError)
             client.terminal
             server.terminal
+            DIY::Logger.info "Error_on_stop flag opened, stopping..."
+            DIY::Logger.info "Dump mac learn table(size is #{@offline.mac_learner.size})... "
+            DIY::Logger.info @offline.mac_learner.dump
             break
           end
-          client,server = @client, @server
+          #~ begin
+            #~ @offline.next_pcap
+            #~ server.terminal
+          #~ rescue EOFError
+            #~ client.terminal
+            #~ server.terminal
+            #~ break
+          #~ end
+          #~ client,server = @client, @server
         rescue EOFError
           client.terminal
           server.terminal
           break
+        ensure
+          #~ client, server = server, client
         end
       end
       DRb.stop_service
@@ -50,6 +67,19 @@ module DIY
       stats_result( end_time - start_time, @fail_count )
     end
     
+    def do_trap
+      Signal.trap("INT") do
+        DIY::Logger.info "bye.."
+        stop
+        exit 0
+      end    
+    end
+    
+    def stop
+      @client.terminal
+      @server.terminal
+    end
+    
     def one_round( client, server, pkts )
       @error_flag = nil
       @round_count = 0 unless @round_count
@@ -58,22 +88,32 @@ module DIY
       if pkts.size >= 10
         DIY::Logger.info "queue size too big: #{pkts.size}, maybe something error"
       end
-      server.ready do |recv_pkt|
-        next if @error_flag # error accur waiting other thread do with it
-        recv_pkt = Packet.new(recv_pkt)
+      
+      recv_pkt_proc_set( pkts )
+      server.ready(&@recv_pkt_proc)
+      
+      client_send(client, pkts)
+      wait_recv_ok(pkts)
+      server.terminal
+    end
+    
+    # 设置回调入口, 由 worker 通过DRb 远程调用
+    def recv_pkt_proc_set(queue)
+      @queue_keeper = queue
+      # 不重新赋值, 防止 DRb 回收
+      @recv_pkt_proc ||= lambda do |recv_pkt|
         begin
-          @strategy.call(pkts.first, recv_pkt, pkts)
+          next if @error_flag # error accur waiting other thread do with it
+          @recv_pkt_keeper = Packet.new(recv_pkt)
+          @strategy.call(@queue_keeper.first, @recv_pkt_keeper, @queue_keeper)
         rescue DIY::UserError =>e
           DIY::Logger.warn("UserError Catch: " + e.inspect)
           e.backtrace.each do |msg|
             DIY::Logger.info(msg)
           end
           @error_flag = e
-        end
+        end        
       end
-      client_send(client, pkts)
-      wait_recv_ok(pkts)
-      server.terminal
     end
     
     def client_send(client, pkts)
@@ -108,9 +148,13 @@ module DIY
       @timeout = timeout
     end
     
+    def error_on_stop(*)
+      @error_on_stop = true
+    end
+    
     def stats_result( cost_time, fail_count )
-      DIY::Logger.info " ====== Finished in #{cost_time} seconds"
-      DIY::Logger.info " ====== #{offline_result}, #{fail_count} failures"
+      DIY::Logger.info " Finished in #{cost_time} seconds"
+      DIY::Logger.info " #{offline_result}, #{fail_count} failures"
     end
     
     def offline_result

data/lib/diy/exceptions.rb

@@ -9,6 +9,10 @@ module DIY
   # 没有报文被指定时
   class ZeroOfflineError < Error; end
   
+  # 报文分解失败
+  class MacLearnConflictError < Error; end
+  class PacketInvalidError < Error; end
+  
   class UserError < Error
     def initialize(real_exception)
       @real_exception = real_exception

data/lib/diy/live.rb

@@ -6,7 +6,7 @@ module DIY
   class Live
     def initialize(device_name)
       DIY::Logger.info( "Initialize Live: #{device_name}" )
-      @live = FFI::PCap::Live.new(:dev=>device_name, :handler => FFI::PCap::CopyHandler, :promisc => true)
+      @live = FFI::PCap::Live.new(:dev=>device_name, :handler => FFI::PCap::CopyHandler, :promisc => true, :timeout=>1)
       DIY::Logger.info( "Listen on:  #{net} " )
       @running = false
       @live.non_blocking= true

data/lib/diy/mac_learner.rb

@@ -1,5 +1,8 @@
 module DIY
   class MacLearner
+    BROAD_MAC = "\377" * 6 # ff:ff:ff:ff:ff:ff
+    GROUP_MAC = 1
+    LEARN_TIME = 60 * 5 # five minutes
     def initialize(default_host = :A)
       @default_host = default_host
       @table = {}
@@ -13,20 +16,71 @@ module DIY
     end
     
     def _learn(mac, where)
-      @table[mac] = where
+      # 除去组播与广播
+      return if mac == BROAD_MAC
+      return if mac[0] & GROUP_MAC == 1
+      set(mac, where)
+    end
+    
+    def get(mac)
+      @table[mac] && @table[mac][0]
+    end
+    
+    def get_time(mac)
+      @table[mac] && @table[mac][1]
+    end
+    
+    def set(mac, where)
+      time_now = Time.now
+      if @table[mac]
+        if @table[mac][0] != where
+          if (time_now - @table[mac][1]) <= LEARN_TIME
+            raise DIY::MacLearnConflictError, "Found mac learn port confict when set #{Utils.pp_mac(mac)} to #{where}"
+          end
+        end
+      end
+      @table[mac] = [ where, time_now ]
+    end
+    
+    def clear(mac)
+      @table[mac] = nil
     end
     
     # 报告包所在的端口 A or B
     # 如果包不在学习表内, 返回缺省端口(默认为A)
     def tellme(packet)
+      valid!(packet)
       src_p = src(packet)
-      if @table.has_key? src_p
-        where =  @table[src_p]
+      dst_p = dst(packet)
+      
+      if src_p == dst_p
+        DIY::Logger.debug("Found SRC mac is the same with DST mac: #{Utils.pp(packet)}")
+        #~ where = @default_host
+        #~ _learn(src_p, where)
+        #~ return where
+      end
+      
+      if src_p != dst_p && get(src_p) && get(src_p) == get(dst_p)
+        #~ if (get_time(src_p) - get_time(dst_p)).abs <= LEARN_TIME
+          #~ DIY::Logger.warn "Found the same mac learner: packet is #{Utils.pp(packet)}"
+          raise DIY::MacLearnConflictError, "Found mac learn port confict"
+        #~ else
+          #~ cls = get_time(src_p) > get_time(dst_p) ? dst_p : src_p
+          #~ clear(cls)
+        #~ end
+      end
+      
+      if get(src_p)
+        where =  get(src_p)
+        _learn( src_p, where )
+      elsif get(dst_p)
+        where = other( get(dst_p) )
+        _learn( src_p, where )
       else
         where = @default_host
-        _learn( src(packet), where )
+        _learn( src_p, where )
       end
-      _learn( dst(packet), other(where) )
+      _learn( dst_p, other(where) )
       where
     end
     
@@ -40,6 +94,20 @@ module DIY
       end
     end
     
+    def size
+      @table.size
+    end
+    
+    def dump
+      ret = "begin dumpping...\n"
+      @table.each do |k, v|
+        mac = Utils.pp_mac(k)
+        ret += "#{mac} -> "
+        ret += "#{v[0]}, "
+        ret += "created at: #{ v[1].strftime("%H:%M:%S") }\n"
+      end
+      ret += "end dump...\n"
+    end
     
     private
     def  src(packet)
@@ -49,5 +117,9 @@ module DIY
     def dst(packet)
       Utils.dst_mac(packet)
     end
+    
+    def valid!(packet)
+      raise DIY::PacketInvalidError, "packet too small" unless packet.size >= 12
+    end
   end
 end

data/lib/diy/offline.rb

@@ -6,6 +6,7 @@ module DIY
     def initialize( pcap_files )
       @pcap_files = [ pcap_files ] if pcap_files.kind_of?(String)
       @pcap_files ||= pcap_files
+      raise ZeroOfflineError, "no files found" if @pcap_files.size == 0
       @off = FFI::PCap::Offline.new(@pcap_files[0])
       
       @ml = MacLearner.new
@@ -14,11 +15,27 @@ module DIY
       @position = 0
       # 记录包在当前文件的位置
       @num = 0
+      # 记录所有包的个数
+      @total = 0
       
       @tmp_pcap = nil
     end
     
+    def mac_learner
+      @ml
+    end
+    
     def nexts
+      begin
+        return _nexts
+      rescue DIY::MacLearnConflictError, DIY::PacketInvalidError =>e
+        DIY::Logger.warn "Found Error when parse #{fullname}: #{e.message}"
+        next_pcap
+        retry
+      end
+    end
+    
+    def _nexts
       ret = []
       # 取一个
       pkt = fetch_one
@@ -32,10 +49,14 @@ module DIY
       
       loop do
         pkt = fetch_one
-        return ret if pkt.nil?
+        return [ret, where] if pkt.nil?
+        #~ if pkt.nil?
+          #~ next_pcap
+          #~ pkt = fetch_one
+        #~ end
         if @ml.tellme(pkt.content) != where
           cached(pkt)
-          return ret
+          return [ret, where]
         else
           ret << pkt
         end
@@ -55,6 +76,7 @@ module DIY
     def next
       pkt = @off.next
       @num += 1
+      @total += 1
       return nil if pkt.nil?
 
       return Packet.new(pkt.copy.body, fullname)
@@ -100,7 +122,7 @@ module DIY
     end
     
     def now_size
-      @num
+      @total
     end
     
     def files_size

data/lib/diy/parser/mu/pcap/ipv4.rb

@@ -81,7 +81,8 @@ class IPv4 < IP
                 when IPPROTO_UDP
                     ipv4.payload = UDP.from_bytes payload
                 when IPPROTO_SCTP
-                    ipv4.payload = SCTP.from_bytes payload
+                    #ipv4.payload = SCTP.from_bytes payload
+                    ipv4.payload = payload
                 else
                     ipv4.payload = payload
                 end

data/lib/diy/strategy_builder.rb

@@ -34,7 +34,7 @@ module DIY
           raise StrategyCallError.new(e)
         else
           if ret == Strategy::OK
-            logger.info("pkt same:")
+            logger.info("pkt same: #{hope_pkt.pretty_print}")
             queue.shift
             return
           elsif ret == Strategy::OK_NO_POP

data/lib/diy/utils.rb

@@ -16,7 +16,7 @@ module DIY
         begin
           new_pkt = pkt.dup
           Mu::Pcap::Ethernet.from_bytes(new_pkt).to_s + size_print_str
-        rescue Mu::Pcap::ParseError =>e
+        rescue Mu::Pcap::ParseError, Exception =>e
           DIY::Logger.debug "parse error from pkt: " + ( pkt[0..10] + "..." ).dump + size_print_str
           return  ( pkt[0..10] + "..." ).dump + size_print_str + "( parse failed )"
         end
@@ -32,6 +32,15 @@ module DIY
         pkt[0..5]
       end
       
+      def pp_mac(mac)
+        raise "MAC MUST BE 6 sizes" unless mac.size == 6
+        begin
+          '%02x:%02x:%02x:%02x:%02x:%02x' % mac.unpack('C6')
+        rescue ArgumentError
+          mac.dump
+        end
+      end
+      
       def wait_until( timeout = 20, &block )
         timeout(timeout) do
           loop do
@@ -54,6 +63,14 @@ module DIY
         new_bt
       end
       
+      def print_backtrace(e)
+        DIY::Logger.info "Dump Exception: #{e.class} -> #{e.message}..."
+        e.backtrace.each do |msg|
+          DIY::Logger.info(msg)
+        end
+        DIY::Logger.info("Dump end!")
+      end
+      
       def ary_match(ary, msg)
         ary.each do |e|
           return true if /#{Regexp.escape(e)}/ === msg

data/lib/diy/version.rb

@@ -1,5 +1,5 @@
 module DIY
   class PCAP
-    VERSION = "0.3.5"
+    VERSION = "0.3.6"
   end
 end

data/lib/diy/worker.rb

@@ -45,10 +45,14 @@ module DIY
           begin
             pkt = @queue.pop
             #~ DIY::Logger.info "callback: #{pkt}"
-            @block.call(pkt) if @block
+            @block.call(pkt) if @start and @block
           rescue DRb::DRbConnError
             DIY::Logger.info "closed connection by controller"
+            @start = false
             @queue.clear
+          rescue RangeError=>e
+            DIY::Utils.print_backtrace(e)
+            raise e
           end
         end
         DIY::Logger.debug "stopped loop callback"
@@ -57,6 +61,7 @@ module DIY
     
     #收包
     def ready(&block)
+      @start = false
       DIY::Logger.info("start recv pkt")
       @block = block
       @queue.clear

data/spec/controller_spec.rb

@@ -131,4 +131,15 @@ describe "Controller" do
     lambda { build2.run }.should_not raise_error  
   end
 
+  it "#run with error_on_stop" do
+    build2 = DIY::Builder.new do
+      use DIY::SimpleStrategy.new
+      filter "not tcp"
+      error_on_stop
+      pcapfiles ["helper/http.pcap", "helper/gre.pcap"]
+      timeout 1
+    end
+    lambda { build2.run }.should_not raise_error    
+  end
+
 end

data/spec/mac_learner_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+describe DIY::MacLearner do
+  
+  before(:each) do
+    @ml = DIY::MacLearner.new
+  end
+  
+  def make_packet(src, dst)
+    src = src * 6
+    dst = dst * 6
+    dst + src
+  end
+  
+  it "#tellme new packet" do
+    pkt = make_packet( "\2", "\4")
+    @ml.tellme(pkt).should == :A
+  end
+  
+  it "#tellme again packet" do
+    pkt = make_packet( "\2", "\4" )
+    @ml.tellme(pkt)
+    @ml.tellme(pkt).should == :A
+    #~ @ml.instance_variable_get("@table").size.should == 1
+    b_pkt = make_packet( "\4" , "\2" )
+    @ml.tellme(b_pkt).should == :B
+    #~ @ml.instance_variable_get("@table").size.should == 2
+    # 另一个新包
+    c_pkt = make_packet( "\6", "\8" )
+    @ml.tellme(c_pkt).should == :A
+    #~ @ml.instance_variable_get("@table").size.should == 3
+    #~ pp @ml.instance_variable_get("@table")
+    # 目标地址选定的
+    d_pkt = make_packet( "\8", "\2")
+    @ml.tellme(d_pkt).should == :B
+    #~ pp @ml.instance_variable_get("@table") 
+    @ml.instance_variable_get("@table")["\8"*6][0].should == :B
+    dd_pkt = make_packet( "\8", "\10")
+    @ml.tellme(dd_pkt).should == :B
+    # 有冲突的
+    e_pkt = make_packet( "\2", "\6" )
+    lambda { @ml.tellme(e_pkt) }.should raise_error(DIY::MacLearnConflictError)
+    # 解决冲突
+    #~ @ml.instance_variable_get("@table")[ "\6"*6 ][1] = Time.now - 10*60
+    #~ @ml.tellme(e_pkt).should == :A
+    #~ @ml.tellme(e_pkt).should == :A
+    
+    # 源目的相同的
+    ee_pkt = make_packet( "\2", "\2")
+    lambda { @ml.tellme(ee_pkt) }.should raise_error(DIY::MacLearnConflictError)
+  end
+  
+  it "#tellme group and broad" do
+    @ml.tellme( make_packet("\2", "\377") ).should == :A
+    @ml.tellme( make_packet( "\377", "\2") ).should == :B
+    @ml.instance_variable_get("@table").size.should == 1
+    @ml.tellme( make_packet("\377", "\4") ).should == :A
+    
+    @ml.tellme( make_packet("\1", "\4")).should == :A
+    @ml.tellme( make_packet("\6", "\1")).should == :A
+    @ml.tellme( make_packet("\4", "\1")).should == :B
+  end
+end

data/spec/offline_spec.rb

@@ -9,35 +9,41 @@ describe DIY::Offline do
     lambda { offline.next_pcap }.should raise_error(DIY::EOFError)
   end
   
+  it "no file" do
+    lambda { DIY::Offline.new([]) }.should raise_error(DIY::ZeroOfflineError)
+  end
+  
   it "should get next special first_pkt" do
     files = [ "helper/app.pcap", "helper/gre.pcap" ]
     offline = DIY::Offline.new(files)
     22.times { offline.nexts }
-    offline.nexts.size.should == 1
+    offline.nexts[0].size.should == 1
   end
   
   it "should get nexts two" do
     files = [ "helper/gre.pcap", "helper/app.pcap" ]
     offline = DIY::Offline.new(files)
-    offline.nexts.size.should == 1
-    offline.nexts.size.should == 2
-    offline.nexts.size.should == 2
+    offline.nexts[0].size.should == 1
+    offline.nexts[0].size.should == 2
+    offline.nexts[0].size.should == 2
     offline.next_pcap
-    offline.nexts.size.should == 1
-    offline.nexts.size.should == 7
+    offline.nexts[0].size.should == 1
+    offline.nexts[0].size.should == 7
     lambda { loop do offline.nexts end }.should raise_error(DIY::EOFError)
   end
   
   it "should get another two" do
     files = [ "helper/http.pcap", "helper/gre.pcap" ]
     offline = DIY::Offline.new(files)
-    offline.nexts.size.should == 1
-    offline.nexts.size.should == 1
-    offline.nexts.size.should == 1
+    offline.nexts[0].size.should == 1
+    offline.nexts[0].size.should == 1
+    offline.nexts[0].size.should == 1
     #change to next
-    offline.nexts.size.should == 1
-    offline.nexts.size.should == 2
-    offline.nexts.size.should == 2
+    a = offline.nexts
+    a[0].size.should == 1
+    a[1].should == :A
+    offline.nexts[0].size.should == 2
+    offline.nexts[0].size.should == 2
     lambda { loop do offline.nexts end }.should raise_error(DIY::EOFError)
   end
   
@@ -46,7 +52,7 @@ describe DIY::Offline do
       600.times do
         files << "helper/http.pcap"
       end
-      puts "files size = #{files.size}"
+      #~ puts "files size = #{files.size}"
       offline = DIY::Offline.new(files)
       lambda {
       loop do

data/spec/utils_spec.rb

@@ -5,6 +5,11 @@ describe DIY::Utils do
     DIY::Utils.pp('a' * 100).should match(/\(100 sizes\)/)
   end
   
+  it "#pp parse error" do
+    badtcp = File.open('helper/badtcp.dat', 'rb') { |io| io.read }
+    lambda { puts DIY::Utils.pp(badtcp) }.should_not raise_error
+  end
+  
   it "#pp false" do
     DIY::Utils.pp('a' * 100, false).should_not match(/\(100 sizes\)/)
   end
@@ -16,5 +21,17 @@ describe DIY::Utils do
   it "#dst_mac" do
     DIY::Utils.dst_mac( 'a' * 100 ).should == "a" * 6
   end
+  
+  it "#pp_mac" do
+    DIY::Utils.pp_mac( "\377" * 6 ).should == "ff:ff:ff:ff:ff:ff"
+  end
+  
+  it "#print_backtrace" do
+    begin
+      raise 
+    rescue 
+      lambda { DIY::Utils.print_backtrace($!) }.should_not raise_error
+    end
+  end
 
 end

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: DIY-pcap
 version: !ruby/object:Gem::Version 
-  hash: 25
-  prerelease: 
+  prerelease: false
   segments: 
   - 0
   - 3
-  - 5
-  version: 0.3.5
+  - 6
+  version: 0.3.6
 platform: ruby
 authors: 
 - yafei Lee
@@ -15,7 +14,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-10-15 00:00:00 Z
+date: 2012-10-22 00:00:00 +08:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: ffi-pcap
@@ -25,7 +25,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 23
         segments: 
         - 0
         - 2
@@ -129,6 +128,7 @@ files:
 - spec/controller_spec.rb
 - spec/device_finder_spec.rb
 - spec/helper/app.pcap
+- spec/helper/badtcp.dat
 - spec/helper/gre.pcap
 - spec/helper/http.pcap
 - spec/helper/long.dat
@@ -142,12 +142,14 @@ files:
 - spec/helper/vlan.dat
 - spec/live_spec.rb
 - spec/logger_spec.rb
+- spec/mac_learner_spec.rb
 - spec/mu_parser_spec.rb
 - spec/offline_spec.rb
 - spec/packet_spec.rb
 - spec/spec_helper.rb
 - spec/utils_spec.rb
 - spec/worker_spec.rb
+has_rdoc: true
 homepage: ""
 licenses: []
 
@@ -161,7 +163,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
@@ -170,14 +171,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: DIY pcap send and recv