bindata 1.6.0 → 1.8.0

data/.gitignore

@@ -1 +1,2 @@
 Gemfile.lock
+wiki

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.8.7
+  - 2.0.0
+  - jruby

data/ChangeLog.rdoc

@@ -1,5 +1,14 @@
 = BinData Changelog
 
+== Version 1.8.0 (2014-01-06)
+
+* Allow custom types to have endian, not just numerics.
+* Added missing field to TCP example.  Thanks to Bertrand Paquet.
+* Made tests compatible with JRuby.  Thanks to Charles Oliver Nutter.
+* Support travis ci for those that use it.  Thanks to Charles Oliver Nutter.
+* Added Buffer for easier handling of nested streams.
+* Added Virtual field.
+
 == Version 1.6.0 (2013-09-02)
 
 * Added license to .gemspec

data/README.md

@@ -28,6 +28,13 @@ r  = Rectangle.read(io)
 puts "Rectangle #{r.name} is #{r.width} x #{r.height}"
 ```
 
+BinData provides a _declarative_ way to read and write structured binary data.
+
+This means the programmer specifies *what* the format of the binary
+data is, and BinData works out *how* to read and write data in this
+format.  It is an easier (and more readable) alternative to
+ruby's `#pack` and `#unpack` methods.
+
 BinData makes it easy to create new data types. It supports all the common
 primitive datatypes that are found in structured binary data formats. Support
 for dependent and variable length fields is built in. 
@@ -42,7 +49,7 @@ for dependent and variable length fields is built in.
 
 # Documentation
 
-[http://bindata.rubyforge.org/manual.html](http://bindata.rubyforge.org/manual.html)
+[Read the wiki](http://github.com/dmendel/bindata/wiki).
 
 -or-
 

data/doc/manual.md

@@ -196,6 +196,31 @@ The increase in clarity can be seen with the above example.  The
 `endian` keyword will cascade to nested types, as illustrated with the
 array in the above example.
 
+The endian keyword can also be used to identify custom types that have
+endianness.  To do this, the class name of the custom types must end with `Le`
+for little endian, and `Be` for big endian.
+
+    class CoordLe < BinData::Record
+      endian :little
+       int16  :x
+      int16  :y
+    end
+
+    class CoordBe < BinData::Record
+      endian :big
+      int16  :x
+      int16  :y
+    end
+
+    class Rectangle < BinData::Record
+      endian :little
+
+      coord  :upper_left
+      coord  :lower_right
+    end
+{:ruby}
+
+
 ## Dependencies between fields
 
 A common occurence in binary file formats is one field depending upon
@@ -409,8 +434,8 @@ There are several parameters that are specific to all primitives.
     both `:assert` and `:value` have the same values.  The following
     are equivalent.
 
-        obj = BinData::Uint32Be.new(:assert => 42, :value => 42)
-        obj = BinData::Uint32Be.new(:asserted_value => 42)
+        obj = BinData::Uint32be.new(:assert => 42, :value => 42)
+        obj = BinData::Uint32be.new(:asserted_value => 42)
     {:ruby}
 
 ## Numerics

data/examples/tcp_ip.rb

@@ -0,0 +1,179 @@
+require 'bindata'
+
+# This is a simple protocol analyser for examining IPv4 packets
+# captured with libpcap on an ethernet network.
+#
+# The dump file can be obtained like this:
+#
+#     sudo tcpdump -i eth0 -s 0 -n -w dump.pcap
+#
+
+
+# Present MAC addresses in a human readable way
+class MacAddr < BinData::Primitive
+  array :octets, :type => :uint8, :initial_length => 6
+
+  def set(val)
+    ints = val.split(/\./).collect { |int| int.to_i }
+    self.octets = ints
+  end
+
+  def get
+    self.octets.collect { |octet| "%02x" % octet }.join(":")
+  end
+end
+
+# Present IP addresses in a human readable way
+class IPAddr < BinData::Primitive
+  array :octets, :type => :uint8, :initial_length => 4
+
+  def set(val)
+    ints = val.split(/\./).collect { |int| int.to_i }
+    self.octets = ints
+  end
+
+  def get
+    self.octets.collect { |octet| "%d" % octet }.join(".")
+  end
+end
+
+# TCP Protocol Data Unit
+class TCP_PDU < BinData::Record
+  mandatory_parameter :packet_length
+
+  endian :big
+
+  uint16 :src_port
+  uint16 :dst_port
+  uint32 :seq
+  uint32 :ack_seq
+  bit4   :doff
+  bit4   :res1
+  bit2   :res2
+  bit1   :urg
+  bit1   :ack
+  bit1   :psh
+  bit1   :rst
+  bit1   :syn
+  bit1   :fin
+  uint16 :window
+  uint16 :checksum
+  uint16 :urg_ptr
+  string :options, :read_length => :options_length_in_bytes
+  string :payload, :read_length => lambda { packet_length - payload.rel_offset }
+
+  def options_length_in_bytes
+    (doff - 5 ) * 4
+  end
+end
+
+# UDP Protocol Data Unit
+class UDP_PDU < BinData::Record
+  mandatory_parameter :packet_length
+
+  endian :big
+
+  uint16 :src_port
+  uint16 :dst_port
+  uint16 :len
+  uint16 :checksum
+  string :payload, :read_length => lambda { packet_length - payload.rel_offset }
+end
+
+# IP Protocol Data Unit
+class IP_PDU < BinData::Record
+  endian :big
+
+  bit4   :version, :asserted_value => 4
+  bit4   :header_length
+  uint8  :tos
+  uint16 :total_length
+  uint16 :ident
+  bit3   :flags
+  bit13  :frag_offset
+  uint8  :ttl
+  uint8  :protocol
+  uint16 :checksum
+  ip_addr :src_addr
+  ip_addr :dest_addr
+  string :options, :read_length => :options_length_in_bytes
+  choice :payload, :selection => :protocol do
+    tcp_pdu  6, :packet_length => :payload_length_in_bytes
+    udp_pdu 17, :packet_length => :payload_length_in_bytes
+    string :default, :read_length => :payload_length_in_bytes
+  end
+
+  def header_length_in_bytes
+    header_length * 4
+  end
+
+  def options_length_in_bytes
+    header_length_in_bytes - options.rel_offset
+  end
+
+  def payload_length_in_bytes
+    total_length - header_length_in_bytes
+  end
+end
+
+# Ethernet Frame - NOTE only ipv4 is supported
+class Ether < BinData::Record
+  IPV4 = 0x0800
+
+  endian :big
+  mac_addr :dst
+  mac_addr :src
+  uint16   :ether_type
+  choice   :payload, :selection => :ether_type do
+    ip_pdu IPV4
+    rest   :default
+  end
+end
+
+class Pcap
+  def initialize(filename)
+    @filename = filename
+  end
+
+  def each_record
+    File.open(@filename) do |io|
+      file = PcapFile.read(io)
+      file.records.each do |rec|
+        yield rec.data
+      end
+    end
+  end
+
+  class PcapFile < BinData::Record
+    endian :little
+
+    struct :head do
+      uint32 :magic
+      uint16 :major
+      uint16 :minor
+      int32  :this_zone
+      uint32 :sig_figs
+      uint32 :snaplen
+      uint32 :linktype
+    end
+
+    array :records, :read_until => :eof do
+      uint32 :ts_sec
+      uint32 :ts_usec
+      uint32 :incl_len
+      uint32 :orig_len
+      string :data, :length => :incl_len
+    end
+  end
+end
+
+require 'pp'
+unless File.exist?('dump.pcap')
+  puts "No dump file found. Create one with: sudo tcpdump -i eth0 -s 0 -n -w dump.pcap"
+  exit 1
+end
+
+cap = Pcap.new('dump.pcap')
+cap.each_record do |str|
+  pp Ether.read(str)
+end

data/lib/bindata.rb

@@ -1,9 +1,10 @@
 # BinData -- Binary data manipulator.
-# Copyright (c) 2007 - 2013 Dion Mendel.
+# Copyright (c) 2007 - 2014 Dion Mendel.
 
 require 'bindata/version'
 require 'bindata/array'
 require 'bindata/bits'
+require 'bindata/buffer'
 require 'bindata/choice'
 require 'bindata/count_bytes_remaining'
 require 'bindata/float'
@@ -16,6 +17,7 @@ require 'bindata/string'
 require 'bindata/stringz'
 require 'bindata/struct'
 require 'bindata/trace'
+require 'bindata/virtual'
 require 'bindata/alignment'
 require 'bindata/deprecated'
 
@@ -30,4 +32,4 @@ require 'bindata/deprecated'
 #
 # BinData is released under the same license as Ruby.
 #
-# Copyright (c) 2007 - 2013 Dion Mendel.
+# Copyright (c) 2007 - 2014 Dion Mendel.

data/lib/bindata/base.rb

@@ -161,7 +161,7 @@ module BinData
 
     # Reads data into this data object.
     def read(io)
-      io = BinData::IO.new(io) unless BinData::IO === io
+      io = BinData::IO::Read.new(io) unless BinData::IO::Read === io
 
       @in_read = true
       clear
@@ -184,7 +184,7 @@ module BinData
 
     # Writes the value for this data object to +io+.
     def write(io)
-      io = BinData::IO.new(io) unless BinData::IO === io
+      io = BinData::IO::Write.new(io) unless BinData::IO::Write === io
 
       do_write(io)
       io.flush

data/lib/bindata/buffer.rb

@@ -0,0 +1,119 @@
+require 'bindata/base'
+require 'bindata/dsl'
+
+module BinData
+  # A Buffer is conceptually a substream within a data stream.  It has a
+  # defined size and it will always read or write the exact number of bytes to
+  # fill the buffer.  Short reads will skip over unused bytes and short writes
+  # will pad the substream with "\0" bytes.
+  #
+  #   require 'bindata'
+  #
+  #   obj = BinData::Buffer.new(:length => 5, :type => [:string, {:value => "abc"}])
+  #   obj.to_binary_s #=> "abc\000\000"
+  #
+  #
+  #   class MyBuffer < BinData::Buffer
+  #     default_parameter :length => 8
+  #
+  #     endian :little
+  #
+  #     uint16 :num1
+  #     uint16 :num2
+  #     # padding occurs here
+  #   end
+  #
+  #   obj = MyBuffer.read("\001\000\002\000\000\000\000\000")
+  #   obj.num1 #=> 1
+  #   obj.num1 #=> 2
+  #   obj.num_bytes #=> 8
+  #
+  #
+  #   class StringTable < BinData::Record
+  #     endian :little
+  #
+  #     uint16 :table_size_in_bytes
+  #     buffer :strings, :length => :table_size_in_bytes do
+  #       array :read_until => :eof do
+  #         uint8 :len
+  #         string :str, :length => :len
+  #       end
+  #     end
+  #   end
+  #     
+  #
+  # == Parameters
+  #
+  # Parameters may be provided at initialisation to control the behaviour of
+  # an object.  These params are:
+  #
+  # <tt>:length</tt>::   The number of bytes in the buffer.
+  # <tt>:type</tt>::     The single type inside the buffer.  Use a struct if
+  #                      multiple fields are required.
+  class Buffer < BinData::Base
+    include DSLMixin
+
+    dsl_parser :buffer
+
+    mandatory_parameters :length, :type
+
+    class << self
+      def arg_extractor
+        MultiFieldArgExtractor
+      end
+
+      def sanitize_parameters!(params) #:nodoc:
+        params.merge!(dsl_params)
+
+        if params.needs_sanitizing?(:type)
+          el_type, el_params = params[:type]
+          params[:type] = params.create_sanitized_object_prototype(el_type, el_params)
+        end
+      end
+    end
+
+    def initialize_instance
+      @type = get_parameter(:type).instantiate(nil, self)
+    end
+
+    def clear? #:nodoc:
+      @type.clear?
+    end
+
+    def assign(val)
+      @type.assign(val)
+    end
+
+    def snapshot
+      @type.snapshot
+    end
+
+    def respond_to?(symbol, include_private = false) #:nodoc:
+      @type.respond_to?(symbol, include_private) || super
+    end
+
+    def safe_respond_to?(symbol, include_private = false) #:nodoc:
+      base_respond_to?(symbol, include_private)
+    end
+
+    def method_missing(symbol, *args, &block) #:nodoc:
+      @type.__send__(symbol, *args, &block)
+    end
+
+    def do_read(io) #:nodoc:
+      io.with_buffer(eval_parameter(:length)) do
+        @type.do_read(io)
+      end
+    end
+
+    def do_write(io) #:nodoc:
+      io.with_buffer(eval_parameter(:length)) do
+        @type.do_write(io)
+      end
+    end
+
+    def do_num_bytes #:nodoc:
+      eval_parameter(:length)
+    end
+  end
+end

data/lib/bindata/dsl.rb

@@ -1,4 +1,40 @@
 module BinData
+  # Extracts args for Records and Buffers.
+  #
+  # Foo.new(:bar => "baz) is ambiguous as to whether :bar is a value or parameter.
+  #
+  # BaseArgExtractor always assumes :bar is parameter.  This extractor correctly
+  # identifies it as value or parameter.
+  class MultiFieldArgExtractor
+    class << self
+      def extract(the_class, the_args)
+        value, parameters, parent = BaseArgExtractor.extract(the_class, the_args)
+
+        if parameters_is_value?(the_class, value, parameters)
+          value = parameters
+          parameters = {}
+        end
+
+        [value, parameters, parent]
+      end
+
+      def parameters_is_value?(the_class, value, parameters)
+        if value.nil? and parameters.length > 0
+          field_names_in_parameters?(the_class, parameters)
+        else
+          false
+        end
+      end
+
+      def field_names_in_parameters?(the_class, parameters)
+        field_names = the_class.fields.field_names
+        param_keys = parameters.keys
+
+        (field_names & param_keys).length > 0
+      end
+    end
+  end
+
   module DSLMixin
     def self.included(base) #:nodoc:
       base.extend ClassMethods
@@ -80,6 +116,8 @@ module BinData
           to_struct_params
         when :array
           to_array_params
+        when :buffer
+          to_array_params
         when :choice
           to_choice_params
         when :primitive
@@ -110,6 +148,8 @@ module BinData
           [:multiple_fields, :optional_fieldnames, :hidden_fields]
         when :array
           [:multiple_fields, :optional_fieldnames]
+        when :buffer
+          [:multiple_fields, :optional_fieldnames]
         when :choice
           [:multiple_fields, :all_or_none_fieldnames, :fieldnames_are_values]
         when :primitive
@@ -157,6 +197,7 @@ module BinData
       def params_from_block(type, &block)
         bindata_classes = {
           :array  => BinData::Array,
+          :buffer => BinData::Buffer,
           :choice => BinData::Choice,
           :struct => BinData::Struct
         }