packet_via_dmem 0.0.8 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0147a2a28aec0b7d80b8e8fc84b3ae62e228893f
-  data.tar.gz: 231b1c57cbb1902621382913c3eab7c9d86e2c57
+  metadata.gz: 87a6a44720c7a589e77bee75c030bda1c59afee4
+  data.tar.gz: 3f05700efab522e4576b99d4c6a51c225d07cac4
 SHA512:
-  metadata.gz: ec74f12df72f23334e86568b15a0e20f66ac7223fbc1d3072ae34d47f619d207353bb69b9d19168db4b071c0a12c4b0ee5cc214e4ce532f6f433b49e69ac9360
-  data.tar.gz: e3772c982460028fd3d4fbc79a78cd1ce8dc4328fdb95db108cceaaeb3f2add2a32da506bd9230b853ab27c8cde1123e376f420022bdb46918ad379787a6e986
+  metadata.gz: 227fd4c042a6d5cd24bc3aa79eb9b3c59a5d4feb4c351bbaed301762ebc2bda825e7c2e1f98fb164e0f879b9f691ab2224e94d526438fbd0a596aeb9d8cb5aa5
+  data.tar.gz: dbd9cb160260210dba193663616b768fb075c404d54ddee39a6968d65501ab36d2c39c169ff5eec9947a241ffb2ebb695db7c1cdcce8783ea15ed73bee528b41

data/README.md

@@ -83,27 +83,22 @@ To capture say packets with IP address 10.11.12.13
 ## Header format
 ### Received header
 
-  * First byte is type of header?
-    * 00 'lu packet' (i.e. whole packet was sent for lookup, i.e. small packet)
-    * 10 'lu packet head' (i.e. only head of packet was sent for lookup, i.e. large packet)
-
-  * Second and third byte appear to tell nothing about where packet came from,
-    but more when it came from. Timing? Counter? Randomness?
-
-  * Fourth byte is 0xf0 on MX80, tendency for last nibble to be 0. Perhaps src fabric stream? If it is zero, we get what seems to be trash (internal stuff?)
-
-  * Fifth byte appears to be source port?
-
-  * Sixth byte is perhaps source NPU? If it is zero, we get what seems to be trash (internal stuff)
-
-  * value of fift+sixth seems to sometime indicate special cases
-    * 0x1fff - Packet missing everything before IPv4 TTL, yet has some extra. I saw BGP from control-plane with this and also TCP/SMB2 with Seq1, it was transit, but perhaps it was via ARP resolve/punt and thus coming from control-plane?
-    * 0x2000 - BFD frames from control-plane or LACP IPv4, if next byte is 1 like below, if next byte is 0 missing MACs too (+5bytes)
-    * 0x4220 - Was LACP MPLS traffic, missing ethertype, MACs changed, 2 mystery bytes
-    * 0x8000 - I need to pop 14 bytes extra
-
-  * 00 (22) (33) (44) \<src\> (66)
-  * 10 (22) (33) (44) \<si\> \<ze\> \<src\> (66)
+  * first four bits is message type
+    * 0 'lu packet' (i.e. whole packet was sent for lookup, i.e. small packet)
+    * 1 'lu packet head' (i.e. only head of packet was sent for lookup, i.e. large packet)
+  * next 13 bits is table entry (memory loc)
+  * next 11 bits is stream
+  * next 3 bits offset (for fabric cell sharing)
+  * next bit is size (17th bit, no way to test, would need >65k packets)
+  * next 16 bits is size (if message type is 1)
+  * next 8 bits is port
+  * next 8 bits is packet type
+
+  * packet type
+    * 0x00 real PITA, looks to be packets from control-plane, but amount of bytes that I need to pop I can't really figure out. I now rely on port# which likely isn't robust. This made me change my mind that 5th byte isn't port, but combined type, as it allowed, what seemed cleaner classification, but unfortunately it is not so.
+    * 0x08 is no-op, i don't need to pop anything, DMAC follows
+    * 0x20 this seems to be quite reliably mpls, i need to do some magic, as we're missing ethertype and my macs are wrong, there is also two extra bytes
+    * 0x80 is some short trash (payload 0xc013c6752759644ae0) no clue what it is
 
 Example from MX960
 
@@ -185,17 +180,6 @@ Example from MX80
     10 07 80 f0 05 b4 81 08
     10 0b 00 f0 02 28 81 08
 
-For this box, second to last byte, divmod 64, returns these ports, which are
-correct port for source.
-
-    TAZ-TBB-0(X vty)# show ixchip ifd
-       IFD       IFD        IX     WAN       Ing Queue     Egr Queue
-      Index      Name       Id     Port      Rt/Ct/Be      H/L
-      ======  ==========  ======  ======  ==============  ======
-       148     ge-1/0/0      2       0        0/32/64       0/32
-       149     ge-1/0/1      2       1        1/33/65       1/33
-       166     ge-1/1/8      2      18       18/50/82      18/50
-
 ### Sent header
 I'm really not sure about sent headers, need more data to figure out what is
 our type. For all my examples, when we sent frame without L2 headers for

data/capture/generate.rb

@@ -25,7 +25,7 @@ class PacketViaDMEM
         when :sent     then sent     << packet.packet.join
         else raise StandardError, "invalid type #{type}"
         end
-        headers   << packet.header.join
+        headers   << packet.header.to_s
         originals << packet.original.join
       end
       File.write('header-'   + name + '.txt', headers.join("\n"))

data/lib/packet_via_dmem/cli.rb

@@ -35,7 +35,7 @@ class PacketViaDMEM
         end
         packet = @opts.original? ? pkt.pretty_original : pkt.pretty_packet
         packet = pkt.pretty pkt.pop(pop) if pop
-        puts '### Packet %d ###' % [count+=1]
+        puts '# Pkt#%d # %s' % [count+=1, pkt.header]
         puts packet
         $stderr.puts pkt.header.join(' ') if @opts.headers?
         puts

data/lib/packet_via_dmem/header/received.rb

@@ -0,0 +1,20 @@
+class PacketViaDMEM
+class Header
+  class Received
+    attr_accessor :msg_type,
+                  :table,
+                  :stream,
+                  :offset,
+                  :size,
+                  :port,
+                  :type
+
+    def to_s
+      ssize = ''
+      ssize = "size: %d # " % size if msg_type > 0
+      "RX: %sport: %d (%s) # strm: %d # type: 0x%x # tbl: %d # o: %d" %
+        [ ssize, port, port.divmod(64).join('/'), stream, type, table, offset ]
+    end
+  end
+end
+end

data/lib/packet_via_dmem/header/sent.rb

@@ -0,0 +1,38 @@
+class PacketViaDMEM
+class Header
+  class Sent
+    attr_accessor :msg_type,
+                  :statistics,
+                  :increment_reference,
+                  :fragment_info,
+                  :drop_hash,
+                  :decrement_reference,
+                  :prequeue_priority,
+                  :offset,
+                  :table,
+                  :color,
+                  :queue_drop_opcode,
+                  :queue_system,
+                  :life,
+                  :queue_number,
+                  :port,
+                  :type
+
+    def to_s
+      type_str = 'unk'
+      type_str = 'wan' if msg_type == 8
+      type_str = 'fab' if msg_type == 0
+      flags = ''
+      flags << (statistics ? 'S' : 's')
+      flags << (increment_reference ? 'I' : 'i')
+      flags << (fragment_info ? 'F' : 'f')
+      flags << (drop_hash ? 'H' : 'h')
+      flags << (decrement_reference ? 'D' : 'd')
+      flags << (prequeue_priority ? 'P' : 'p')
+      "TX: %s # %s # P:%d(%s) # Ty:%x # Q(%d@%d) # C:%d # D:%d # O:%d # T:%d # L:%d" %
+        [ type_str, flags, port, port.divmod(64).join('/'), type, queue_number,
+          queue_system, color, queue_drop_opcode, offset, table, life ]
+    end
+  end
+end
+end

data/lib/packet_via_dmem/packet.rb

@@ -2,18 +2,18 @@ class PacketViaDMEM
 
   class Packet
     class NoPayload < Error; end
-    attr_reader :type, :packet, :header, :original
+    attr_reader :type, :packet, :header, :popped, :original
 
     def pop bytes
       @original[bytes..-1]
     end
 
-    def header_and_packet packet, pop_bytes, push_bytes
-      header  = packet[0..pop_bytes-1]
+    def popped_and_packet packet, pop_bytes, push_bytes
+      popped  = packet[0..pop_bytes-1]
       payload = packet[pop_bytes..-1]
       raise NoPayload, "no payload for #{packet}" unless payload
       payload = push_bytes + payload
-      [header, payload]
+      [popped, payload]
     end
 
     def to_s
@@ -49,6 +49,61 @@ class PacketViaDMEM
       out
     end
 
+    private
+
+    def get_pop_push pkt, type, port, macs
+      pop, push = 0, []
+      case type
+      when *Type::MPLS
+        pop, push = general_pop_push(pkt, pop, macs, FAKE[:etype_mpls])
+      when *Type::SELF
+        pop, push = self_pop_push(pkt, port, macs, pop)
+      when *Type::NOPOP
+        # no op, DMAC follows
+      end
+      [pop, push]
+    end
+
+    def general_pop_push pkt, pop, macs, ether_type
+      pop += 2
+      if macs
+        pop+=12
+        push = FAKE[:dmac] + FAKE[:smac] + ether_type
+        #push = pkt[2..13] + ether_type
+        [pop, push]
+      else
+        pop+=3
+        push = FAKE[:dmac] + FAKE[:smac] + ether_type
+        [pop, push]
+      end
+    end
+
+    def self_pop_push pkt, port, macs, pop
+      push = []
+      case port
+      when 0x80
+        pop+=14
+      when 0x1f
+        pop+=7
+        push = FAKE[:dmac] + FAKE[:smac] + FAKE[:etype_ipv4] + FAKE[:ipv4]
+      else
+        if macs and port == 0x20
+          pop+=23
+        else
+          pop, push = general_pop_push(pkt, pop, macs, FAKE[:etype_ipv4])
+        end
+      end
+      [pop, push]
+    end
+
+
+
+    module Type
+      SELF  = [ 0x00, 0x40 ]
+      NOPOP = [ 0x08, 0x80 ]
+      MPLS  = [ 0x20 ]
+    end
+
 
   end
 end

data/lib/packet_via_dmem/received.rb

@@ -1,70 +1,38 @@
+require_relative 'header/received'
 class PacketViaDMEM
   class Received < Packet
 
     def initialize packet, debug
-      @debug           = debug
-      @type            = :received
-      @original        = packet
-      @header, @packet = parse_packet packet
+      @debug                    = debug
+      @type                     = :received
+      @original                 = packet.dup
+      @header                   = Header::Received.new
+      @popped, @packet          = parse_packet packet
     end
 
     private
 
     def parse_packet pkt
-      pop, push = 6, []
-      offset = 0
-      case pkt.first.to_i(16)
-      when 0x00 then offset = 0 #1,2,3,4,5,6
-      when 0x10 then offset = 2 #1,2,3,4,7,8,5,6
-      end
-      pop += offset
-      type = pkt[4+offset..5+offset].join.to_i(16)
-      macs = pkt[6+offset].to_i(16) > 0 # macs, maybe...
-      case type
-      # these were self originated
-      when 0x8000
-        pop+=14
-      # ae/802.1AX is special, I seem to have 2 bytes I don't know
-      # and ethertype missing, and MAC is weird, mpls labels are present
-      # i'd need example carrying IPv4/IPv6 instead of MPLS to decide those two bytes
-      when *MAGIC::MPLS
-        pop, push = get_pop_push(pkt, pop, offset, macs, FAKE[:etype_mpls])
-      when *MAGIC::IPV4 # these were BFD packets from control-plane
-        pop, push = get_pop_push(pkt, pop, offset, macs, FAKE[:etype_ipv4])
-      # some BGP packets were like this
-      # also SMB2 TCP Seq1 (maybe post ARP from control-plane?)
-      # they are misssing all of ipv4 headers before TTL
-      when 0x1f00
-        pop+=7
-        push = FAKE[:dmac] + FAKE[:smac] + FAKE[:etype_ipv4] + FAKE[:ipv4]
-      when *MAGIC::NOPOP
-        # no-op, DMAC follows immedately
-      else
-        $stderr.puts "unknown type: 0x#{type.to_s(16)}" if @debug
-      end
-      header_and_packet pkt, pop, push
-    end
+      head = pkt.shift(4).join.to_i(16)
 
-    def get_pop_push pkt, pop, offset, macs, ether_type
-      if macs
-        pop+=14 #pop macs and weird two bytes (return macs in push)
-        push = pkt[8+offset..19+offset] + ether_type
-        [pop, push]
-      else
-        pop+=5
-        push = FAKE[:dmac] + FAKE[:smac] + ether_type
-        [pop, push]
-      end
-    end
+      @header.msg_type = (head & 0xf0000000) >> 28
+      @header.table    = (head & 0xfff8000)  >> 15
+      @header.stream   = (head & 0x7ff0)     >> 4
+      @header.offset   = (head & 0xe)        >> 1
+      @header.size     = (head & 0x1)        << 16
+
+      @header.size    += pkt.shift(2).join.to_i(16) if @header.msg_type == PACKET_HEAD
+      @header.port     = pkt.shift.to_i(16)
+      @header.type     = pkt.shift.to_i(16)
 
-    module MAGIC
-      MPLS = [ 0x4220 ]
-      IPV4 = [ 0x2000 ]
-      # 4008, 8008, 8108 were ETH, MPLS, IPV4
-      # 9208 was ETH, IPv4, UDP, IPSEC/ESP
-      # 4108 was ETH, IPv4, UDP, BFD
-      # b080 was unknown just 9 bytes after header (c013c6752759644ae0)
-      NOPOP = [ 0x4008, 0x4108, 0x8008, 0x8108, 0x9208, 0xb080 ]
+      pop, push = 0, []
+      macs = pkt.first.to_i(16) > 0 # macs, maybe...
+
+      pop, push = get_pop_push pkt, @header.type, @header.port, macs
+      popped_and_packet pkt, pop, push
     end
+
+    PACKET = 0
+    PACKET_HEAD = 1
   end
 end

data/lib/packet_via_dmem/sent.rb

@@ -1,31 +1,52 @@
+require_relative 'header/sent'
 class PacketViaDMEM
   class Sent < Packet
 
     def initialize packet, debug
       @debug           = debug
       @type            = :sent
-      @original        = packet
-      @header, @packet = parse_packet packet
+      @original        = packet.dup
+      @header          = Header::Sent.new
+      @popped, @packet = parse_packet packet
     end
 
     private
 
     def parse_packet pkt
-      pop, push = nil, []
-      case pkt.first.to_i(16)
-      when 0x00 # we're sending to fabric
-        # we may send MAC to fabric,byte 6, 7, 9, 11, 21?
-        if pkt[5].to_i(16) == 0xf0 # we don't send MAC to fabric
-          push = FAKE[:dmac] + FAKE[:smac] + FAKE[:etype_mpls]
-          pop = 24
-        else # we send MAC to fabric
-          pop = 33
-        end
-      when 0x08 then pop = 13
-      else pop = @sent
+      head = pkt.shift(4).join.to_i(16)
+      @header.msg_type            = (head & 0xffffffff) >> 28
+      @header.statistics          = head[27] == 1
+      @header.increment_reference = head[26] == 1
+      @header.fragment_info       = head[25] == 1
+      @header.drop_hash           = head[24] == 1
+      @header.decrement_reference = head[23] == 1
+      @header.prequeue_priority   = head[22] == 1
+      @header.offset              = (head & 0x3fffff) >> 13
+      @header.table               = (head & 0x1fff)
+
+      head = pkt.shift(4).join.to_i(16)
+      raise Packet::NoPayload, "#{@original.join(' ')} had no payload" unless pkt[0]
+      @header.color               = (head & 0x7fffffff) >> 29
+      @header.queue_drop_opcode   = (head & 0xcffffff)  >> 27
+      @header.queue_system        = (head & 0x3ffffff)  >> 24
+      @header.life                = (head & 0x7fffff)   >> 21
+      @header.queue_number        = (head & 0x1fffff)
+
+      pop, push = 0, []
+      @header.port = pkt.shift.to_i(16)
+      @header.type = pkt.shift.to_i(16)
+      macs = pkt.first.to_i(16) > 0 # macs, maybe..
+      # uuhhohh, msg_type is always 
+      if not @header.statistics
+        pop, push = get_pop_push pkt, @header.type, @header.port, macs
+      else
+        pop = 3
       end
-      header_and_packet pkt, pop, push
+      popped_and_packet pkt, pop, push
     end
 
+    FABRIC = 0
+    WAN    = 8
+
   end
 end

data/packet_via_dmem.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name              = 'packet_via_dmem'
-  s.version           = '0.0.8'
+  s.version           = '0.0.10'
   s.licenses          = %w( Apache-2.0 )
   s.platform          = Gem::Platform::RUBY
   s.authors           = [ 'Saku Ytti' ]

data/spec/packet_via_dmem_spec.rb

@@ -23,7 +23,7 @@ describe PacketViaDMEM do
       when 'received' then @received[name] = File.read(entry).split
       when 'sent'     then @sent[name]     = File.read(entry).split
       when 'original' then @original[name] = File.read(entry).split
-      when 'header'   then @header[name]   = File.read(entry).split
+      when 'header'   then @header[name]   = File.read(entry).split "\n"
       end
     end
   end
@@ -43,7 +43,7 @@ describe PacketViaDMEM do
     it 'produces correct headers' do
       @parsed.each do |name, packets|
         packets.each_with_index do |packet, index|
-          packet.header.join.must_equal @header[name][index], "header #{name} at line #{index+1}"
+          packet.header.to_s.must_equal @header[name][index], "header #{name} at line #{index+1}"
         end
         packets.size.must_equal @header[name].size, "header #{name} has incorrect amount of packets"
       end
@@ -76,7 +76,7 @@ describe PacketViaDMEM do
         packets.each_with_index do |packet, index|
           packet.original.join.must_equal @original[name][index], "original #{name} at line #{index+1}"
         end
-        packets.size.must_equal @header[name].size, "original #{name} has incorrect amount of packets"
+        packets.size.must_equal @original[name].size, "original #{name} has incorrect amount of packets"
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: packet_via_dmem
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.10
 platform: ruby
 authors:
 - Saku Ytti
@@ -41,6 +41,8 @@ files:
 - capture/generate.rb
 - lib/packet_via_dmem.rb
 - lib/packet_via_dmem/cli.rb
+- lib/packet_via_dmem/header/received.rb
+- lib/packet_via_dmem/header/sent.rb
 - lib/packet_via_dmem/packet.rb
 - lib/packet_via_dmem/packets.rb
 - lib/packet_via_dmem/received.rb