jbangert-bindata 1.5.0

data/examples/NBT.txt

@@ -0,0 +1,149 @@
+Named Binary Tag specification
+
+NBT (Named Binary Tag) is a tag based binary format designed to carry large amounts of binary data with smaller amounts of additional data.
+An NBT file consists of a single GZIPped Named Tag of type TAG_Compound.
+
+A Named Tag has the following format:
+
+    byte tagType
+    TAG_String name
+    [payload]
+    
+The tagType is a single byte defining the contents of the payload of the tag.
+
+The name is a descriptive name, and can be anything (eg "cat", "banana", "Hello World!"). It has nothing to do with the tagType.
+The purpose for this name is to name tags so parsing is easier and can be made to only look for certain recognized tag names.
+Exception: If tagType is TAG_End, the name is skipped and assumed to be "".
+
+The [payload] varies by tagType.
+
+Note that ONLY Named Tags carry the name and tagType data. Explicitly identified Tags (such as TAG_String above) only contains the payload. 
+
+
+The tag types and respective payloads are:
+
+    TYPE: 0  NAME: TAG_End
+    Payload: None.
+    Note:    This tag is used to mark the end of a list.
+             Cannot be named! If type 0 appears where a Named Tag is expected, the name is assumed to be "".
+             (In other words, this Tag is always just a single 0 byte when named, and nothing in all other cases)
+    
+    TYPE: 1  NAME: TAG_Byte
+    Payload: A single signed byte (8 bits)
+
+    TYPE: 2  NAME: TAG_Short
+    Payload: A signed short (16 bits, big endian)
+
+    TYPE: 3  NAME: TAG_Int
+    Payload: A signed short (32 bits, big endian)
+
+    TYPE: 4  NAME: TAG_Long
+    Payload: A signed long (64 bits, big endian)
+
+    TYPE: 5  NAME: TAG_Float
+    Payload: A floating point value (32 bits, big endian, IEEE 754-2008, binary32)
+
+    TYPE: 6  NAME: TAG_Double
+    Payload: A floating point value (64 bits, big endian, IEEE 754-2008, binary64)
+    
+    TYPE: 7  NAME: TAG_Byte_Array
+    Payload: TAG_Int length 
+             An array of bytes of unspecified format. The length of this array is <length> bytes
+
+    TYPE: 8  NAME: TAG_String
+    Payload: TAG_Short length 
+             An array of bytes defining a string in UTF-8 format. The length of this array is <length> bytes
+
+    TYPE: 9  NAME: TAG_List
+    Payload: TAG_Byte tagId
+             TAG_Int length
+             A sequential list of Tags (not Named Tags), of type <typeId>. The length of this array is <length> Tags
+    Notes:   All tags share the same type.
+             
+    TYPE: 10 NAME: TAG_Compound
+    Payload: A sequential list of Named Tags. This array keeps going until a TAG_End is found.
+             TAG_End end
+    Notes:   If there's a nested TAG_Compound within this tag, that one will also have a TAG_End, so simply reading until the next TAG_End will not work.
+             The names of the named tags have to be unique within each TAG_Compound
+             The order of the tags is not guaranteed.
+             
+             
+
+
+
+Decoding example:
+(Use http://www.minecraft.net/docs/test.nbt to test your implementation)
+
+
+First we start by reading a Named Tag.
+After unzipping the stream, the first byte is a 10. That means the tag is a TAG_Compound (as expected by the specification).
+
+The next two bytes are 0 and 11, meaning the name string consists of 11 UTF-8 characters. In this case, they happen to be "hello world".
+That means our root tag is named "hello world". We can now move on to the payload.
+
+From the specification, we see that TAG_Compound consists of a series of Named Tags, so we read another byte to find the tagType.
+It happens to be an 8. The name is 4 letters long, and happens to be "name". Type 8 is TAG_String, meaning we read another two bytes to get the length,
+then read that many bytes to get the contents. In this case, it's "Bananrama".
+
+So now we know the TAG_Compound contains a TAG_String named "name" with the content "Bananrama"
+
+We move on to reading the next Named Tag, and get a 0. This is TAG_End, which always has an implied name of "". That means that the list of entries
+in the TAG_Compound is over, and indeed all of the NBT file.
+
+So we ended up with this:
+
+	TAG_Compound("hello world"): 1 entries
+	{
+	   TAG_String("name"): Bananrama
+	}
+
+
+
+For a slightly longer test, download http://www.minecraft.net/docs/bigtest.nbt
+You should end up with this:
+
+	TAG_Compound("Level"): 11 entries
+	{
+	   TAG_Short("shortTest"): 32767
+	   TAG_Long("longTest"): 9223372036854775807
+	   TAG_Float("floatTest"): 0.49823147
+	   TAG_String("stringTest"): HELLO WORLD THIS IS A TEST STRING ���!
+	   TAG_Int("intTest"): 2147483647
+	   TAG_Compound("nested compound test"): 2 entries
+	   {
+	      TAG_Compound("ham"): 2 entries
+	      {
+	         TAG_String("name"): Hampus
+	         TAG_Float("value"): 0.75
+	      }
+	      TAG_Compound("egg"): 2 entries
+	      {
+	         TAG_String("name"): Eggbert
+	         TAG_Float("value"): 0.5
+	      }
+	   }
+	   TAG_List("listTest (long)"): 5 entries of type TAG_Long
+	   {
+	      TAG_Long: 11
+	      TAG_Long: 12
+	      TAG_Long: 13
+	      TAG_Long: 14
+	      TAG_Long: 15
+	   }
+	   TAG_Byte("byteTest"): 127
+	   TAG_List("listTest (compound)"): 2 entries of type TAG_Compound
+	   {
+	      TAG_Compound: 2 entries
+	      {
+	         TAG_String("name"): Compound tag #0
+	         TAG_Long("created-on"): 1264099775885
+	      }
+	      TAG_Compound: 2 entries
+	      {
+	         TAG_String("name"): Compound tag #1
+	         TAG_Long("created-on"): 1264099775885
+	      }
+	   }
+	   TAG_Byte_Array("byteArrayTest (the first 1000 values of (n*n*255+n*7)%100, starting with n=0 (0, 62, 34, 16, 8, ...))"): [1000 bytes]
+	   TAG_Double("doubleTest"): 0.4931287132182315
+	}

data/examples/gzip.rb

@@ -0,0 +1,161 @@
+require 'bindata'
+require 'forwardable'
+
+# An example of a reader / writer for the GZIP file format as per rfc1952.
+# Note that compression is not implemented to keep the example small.
+class Gzip
+  extend Forwardable
+
+  # Known compression methods
+  DEFLATE = 8
+
+  class Extra < BinData::Record
+    endian :little
+
+    uint16 :len,  :length => lambda { data.length }
+    string :data, :read_length => :len
+  end
+
+  class Header < BinData::Record
+    endian :little
+
+    uint16  :ident,      :value => 0x8b1f, :check_value => 0x8b1f
+    uint8   :compression_method, :initial_value => DEFLATE
+
+    bit3    :freserved,  :value => 0, :check_value => 0
+    bit1    :fcomment,   :value => lambda { comment.length > 0 ? 1 : 0 }
+    bit1    :ffile_name, :value => lambda { file_name.length > 0 ? 1 : 0 }
+    bit1    :fextra,     :value => lambda { extra.len > 0 ? 1 : 0 }
+    bit1    :fcrc16,     :value => 0  # see comment below
+    bit1    :ftext
+
+    # Never include header crc.  This is because the current versions of the
+    # command-line version of gzip (up through version 1.3.x) do not
+    # support header crc's, and will report that it is a "multi-part gzip
+    # file" and give up.
+
+    uint32  :mtime
+    uint8   :extra_flags
+    uint8   :os,         :initial_value => 255   # unknown OS
+
+    # These fields are optional depending on the bits in flags
+    extra   :extra,      :onlyif => lambda { fextra.nonzero? }
+    stringz :file_name,  :onlyif => lambda { ffile_name.nonzero? }
+    stringz :comment,    :onlyif => lambda { fcomment.nonzero? }
+    uint16  :crc16,      :onlyif => lambda { fcrc16.nonzero? }
+  end
+
+  class Footer < BinData::Record
+    endian :little
+
+    uint32 :crc32
+    uint32 :uncompressed_size
+  end
+
+  def initialize
+    @header = Header.new
+    @footer = Footer.new
+  end
+
+  attr_accessor :compressed
+  def_delegators :@header, :file_name=, :file_name
+  def_delegators :@header, :comment=, :comment
+  def_delegators :@header, :compression_method
+  def_delegators :@footer, :crc32, :uncompressed_size
+
+  def mtime
+    Time.at(@header.mtime.snapshot)
+  end
+
+  def mtime=(tm)
+    @header.mtime = tm.to_i
+  end
+
+  def total_size
+    @header.num_bytes + @compressed.size + @footer.num_bytes
+  end
+
+  def compressed_data
+    @compressed
+  end
+
+  def set_compressed_data(compressed, crc32, uncompressed_size)
+    @compressed               = compressed
+    @footer.crc32             = crc32
+    @footer.uncompressed_size = uncompressed_size
+  end
+
+  def read(file_name)
+    File.open(file_name, "r") do |io|
+      @header.read(io)
+
+      # Determine the size of the compressed data.  This is needed because
+      # we don't actually uncompress the data.  Ideally the uncompression
+      # method would read the correct number of bytes from the IO and the
+      # IO would be positioned ready to read the footer.
+
+      pos = io.pos
+      io.seek(-@footer.num_bytes, IO::SEEK_END)
+      compressed_size = io.pos - pos
+      io.seek(pos)
+
+      @compressed = io.read(compressed_size)
+      @footer.read(io)
+    end
+  end
+
+  def write(file_name)
+    File.open(file_name, "w") do |io|
+      @header.write(io)
+      io.write(@compressed)
+      @footer.write(io)
+    end
+  end
+end
+
+if __FILE__ == $0
+  # Write a gzip file.
+  print "Creating a gzip file ... "
+  g = Gzip.new
+  # Uncompressed data is "the cat sat on the mat"
+  g.set_compressed_data("+\311HUHN,Q(\006\342\374<\205\022 77\261\004\000",
+                        3464689835, 22)
+  g.file_name = "poetry"
+  g.mtime = Time.now
+  g.comment = "A stunning piece of prose"
+  g.write("poetry.gz")
+  puts "done."
+  puts
+
+  # Read the created gzip file.
+  print "Reading newly created gzip file ... "
+  g = Gzip.new
+  g.read("poetry.gz")
+  puts "done."
+  puts
+
+  puts "Printing gzip file details in the format of gzip -l -v"
+
+  # compression ratio
+  ratio = 100.0 * (g.uncompressed_size - g.compressed.size) /
+            g.uncompressed_size
+
+  comp_meth = (g.compression_method == Gzip::DEFLATE) ? "defla" : ""
+
+  # Output using the same format as gzip -l -v
+  puts "method  crc     date  time           compressed        " +
+       "uncompressed  ratio uncompressed_name"
+  puts "%5s %08x %6s %5s %19s %19s %5.1f%% %s"  % [comp_meth,
+                                                   g.crc32,
+                                                   g.mtime.strftime('%b %d'),
+                                                   g.mtime.strftime('%H:%M'),
+                                                   g.total_size,
+                                                   g.uncompressed_size,
+                                                   ratio,
+                                                   g.file_name]
+  puts "Comment: #{g.comment}" if g.comment != ""
+  puts
+
+  puts "Executing gzip -l -v"
+  puts `gzip -l -v poetry.gz`
+end

data/examples/ip_address.rb

@@ -0,0 +1,22 @@
+require 'bindata'
+
+# A custom type representing an IP address.
+# The underlying binary representation is a sequence of four octets.
+# The human accessible representation is a dotted quad.
+class IPAddr < BinData::Primitive
+  array :octets, :type => :uint8, :initial_length => 4
+
+  def set(val)
+    ints = val.split(/\./).collect { |int| int.to_i }
+    self.octets = ints
+  end
+
+  def get
+    self.octets.collect { |octet| "%d" % octet }.join(".")
+  end
+end
+
+ip = IPAddr.new("127.0.0.1")
+
+puts "human readable value:  #{ip}"                     #=> 127.0.0.1
+puts "binary representation: #{ip.to_binary_s.inspect}" #=> "\177\000\000\001"

data/examples/list.rb

@@ -0,0 +1,124 @@
+require 'bindata'
+
+# An example of a recursively defined data format.
+#
+# This example format describes atoms and lists.
+# It is recursive because lists can contain other lists.
+#
+# Atoms - contain a single integer
+# Lists - contain a mixture of atoms and lists
+#
+# The binary representation is:
+#
+# Atoms - A single byte 'a' followed by an int32 containing the value.
+# Lists - A single byte 'l' followed by an int32 denoting the number of
+#         items in the list.  This is followed by all the items in the list.
+#
+# All integers are big endian.
+#
+#
+# A first attempt at a declaration would be:
+#
+#     class Atom < BinData::Record
+#       string  :tag, :length => 1, :check_value => 'a'
+#       int32be :val
+#     end
+#
+#     class List < BinData::Record
+#       string  :tag,  :length => 1, :check_value => 'l'
+#       int32be :num,  :value => lambda { vals.length }
+#       array   :vals, :initial_length => :num do
+#         choice :selection => ??? do
+#           atom
+#           list
+#         end
+#       end
+#     end
+#
+# Notice how we get stuck on attemping to write a declaration for
+# the contents of the list.  We can't determine if the list item is
+# an atom or list because we haven't read it yet.  It appears that
+# we can't proceed.
+#
+# The cause of the problem is that the tag identifying the type is
+# coupled with that type.
+#
+# The solution is to decouple the tag from the type.  We introduce a
+# new type 'Term' that is a thin container around the tag plus the
+# type (atom or list).
+#
+# The declaration then becomes:
+#
+#     class Term < BinData::Record; end  # forward declaration
+#     
+#     class Atom < BinData::Int32be
+#     end
+#     
+#     class List < BinData::Record
+#       int32be :num,  :value => lambda { vals.length }
+#       array   :vals, :type => :term, :initial_length => :num
+#     end
+#     
+#     class Term < BinData::Record
+#       string :tag, :length => 1
+#       choice :term, :selection => :tag do
+#         atom 'a'
+#         list 'l'
+#       end
+#     end
+
+
+class Term < BinData::Record; end  # Forward declaration
+
+class Atom < BinData::Int32be
+  def decode
+    snapshot
+  end
+
+  def self.encode(val)
+    Atom.new(val)
+  end
+end
+
+class List < BinData::Record
+  int32be :num,  :value => lambda { vals.length }
+  array   :vals, :initial_length => :num, :type => :term
+
+  def decode
+    vals.collect { |v| v.decode }
+  end
+
+  def self.encode(val)
+    List.new(:vals => val.collect { |v| Term.encode(v) })
+  end
+end
+
+class Term < BinData::Record
+  string :tag, :length => 1
+  choice :term, :selection => :tag do
+    atom 'a'
+    list 'l'
+  end
+
+  def decode
+    term.decode
+  end
+
+  def self.encode(val)
+    if Fixnum === val
+      Term.new(:tag => 'a', :term => Atom.encode(val))
+    else
+      Term.new(:tag => 'l', :term => List.encode(val))
+    end
+  end
+end
+
+
+puts "A single Atom"
+p Term.encode(4)
+p Term.encode(4).decode
+puts
+
+puts "A nested List"
+p Term.encode([1, [2, 3], 4])
+p Term.encode([1, [2, 3], 4]).decode