cbor-diag 0.1.0

data/lib/cbor-diagnostic.rb

@@ -0,0 +1,74 @@
+# This should work with the C-ext cbor-ruby as well as with our cbor-pure
+unless defined?(CBOR)
+  require_relative 'cbor-pure'
+end
+
+class Object
+  def cbor_diagnostic
+    inspect
+  end
+end
+
+class NilClass
+  def cbor_diagnostic
+    "null"
+  end
+end
+
+class Float
+  def cbor_diagnostic           # do a little bit of JSON.stringify gaming (ECMA-262, 9.8.1)
+    a = abs
+    if a < 1 && a >= 1e-6
+      inspect.sub(/(\d)[.](\d+)e-(\d+)/) {"0.#{"0" * ($3.to_i - 1)}#{$1}#{$2}"}
+    else
+      inspect.sub(/(e[+-])0+/) {$1}
+    end
+  end
+end
+
+raise unless 0.00006103515625.cbor_diagnostic == "0.00006103515625"
+raise unless 0.99.cbor_diagnostic == "0.99"
+raise unless 0.099.cbor_diagnostic == "0.099"
+raise unless 0.0000099.cbor_diagnostic == "0.0000099"
+
+class String
+  unless String.instance_methods.include?(:b)
+    def b
+      dup.force_encoding(Encoding::BINARY)
+    end
+  end
+  def hexbytes(sep = '')
+    bytes.map{|x| "%02x" % x}.join(sep)
+  end
+  def cbor_diagnostic
+    if lengths = cbor_stream?
+      pos = 0
+      "(_ #{lengths.map{|l| r = self[pos, l].cbor_diagnostic; pos += l; r}.join(", ")})"
+    else
+      if encoding == Encoding::BINARY
+        "h'#{hexbytes}'"
+      else
+        inspect.encode(Encoding::UTF_16BE).bytes.each_slice(2).map {
+          |c1, c2| c = (c1 << 8)+c2; c < 128 ? c.chr : '\u%04x' % c }.join
+      end
+    end
+  end
+end
+
+class Array
+  def cbor_diagnostic
+    "[#{"_ " if cbor_stream?}#{map(&:cbor_diagnostic).join(", ")}]"
+  end
+end
+
+class Hash
+  def cbor_diagnostic
+    "{#{"_ " if cbor_stream?}#{map{ |k, v| %{#{k.cbor_diagnostic}: #{v.cbor_diagnostic}}}.join(", ")}}"
+  end
+end
+
+class CBOR::Tagged
+  def cbor_diagnostic
+    "#{tag}(#{data.cbor_diagnostic})"
+  end
+end

data/lib/cbor-pretty.rb

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+
+# This should work with the C-ext cbor-ruby as well as with our cbor-pure
+unless defined?(CBOR)
+  require_relative 'cbor-pure'
+end
+
+class String
+  def hexbytes(sep = '')
+    bytes.map{|x| "%02x" % x}.join(sep)
+  end
+end
+
+
+class CBOR
+  def self.pretty(s, indent = 0, max_target = 40)
+    new(s).pretty_item_final(indent, max_target)
+  end
+
+  def take_and_print(n, prefix = '')
+    s = take(n)
+    @out << prefix
+    @out << s.hexbytes
+    s
+  end
+  
+  def pretty_item_streaming(ib)
+    res = nil
+    @out << " # #{MT_NAMES[ib >> 5]}(*)\n"
+    @indent += 1
+    case ib >>= 5
+    when 2, 3, 4, 5
+      while (element = pretty_item) != BREAK
+      end
+    when 7; res = BREAK
+    else raise "unknown ib #{ib} for additional information 31"
+    end
+    @indent -= 1
+    res
+  end
+
+  MT_NAMES = ["unsigned", "negative", "bytes", "text", "array", "map", "tag", "primitive"]
+
+  def pretty_item
+    ib = take_and_print(1, '   ' * @indent).ord
+    ai = ib & 0x1F
+    val = case ai
+          when 0...24; ai
+          when 24; take_and_print(1, ' ').ord
+          when 25; take_and_print(2, ' ').unpack("n").first
+          when 26; (s = take_and_print(4, ' ')).unpack("N").first
+          when 27; (s = take_and_print(8, ' ')).unpack("Q>").first
+          when 31; return pretty_item_streaming(ib)
+          else raise "unknown additional information #{ai} in ib #{ib}"
+          end
+    @out << " # #{MT_NAMES[ib >> 5]}(#{val})\n"
+    @indent += 1
+    case ib >>= 5
+    when 6
+      pretty_item
+    when 2, 3
+      @out << '   ' * (@indent)
+      s = take_and_print(val)
+      @out << " # #{s.inspect}"
+      @out << "\n"
+    when 4; val.times { pretty_item }
+    when 5; val.times { pretty_item; pretty_item}
+    end
+    @indent -= 1
+    nil
+  end
+
+  def pretty_item_final(indent = 0, max_target = 40)
+    @out = ''
+    @indent = indent
+    pretty_item
+    raise if @pos != @buffer.size
+    target = [@out.each_line.map {|ln| ln =~ /#/ || 0}.max, max_target].min
+    @out.each_line.map {|ln|
+      col = ln =~ /#/
+      if col && col < target
+        ln[col, 0] = ' ' * (target - col)
+      end
+      ln
+    }.join
+  end
+
+end

data/lib/cbor-pure.rb

@@ -0,0 +1,260 @@
+# -*- coding: utf-8 -*-
+
+require_relative "half.rb"
+
+
+class CBOR
+  module Streaming
+    def cbor_stream?
+      @cbor_streaming
+    end
+    def cbor_stream!(b = true)
+      @cbor_streaming = b
+      self
+    end
+  end
+  Array.send(:include, Streaming)
+  Hash.send(:include, Streaming)
+  String.send(:include, Streaming)
+
+  class Break
+  end
+  BREAK = Break.new.freeze
+
+  Tagged = Struct.new(:tag, :data) do
+    def to_s
+      "#{tag}(#{data})"
+    end
+    def inspect
+      "#{tag}(#{data.inspect})"
+    end
+  end
+
+  TAG_BIGNUM_BASE = 2
+
+  Simple = Struct.new(:value) do
+    def to_s
+      if value == 23
+        "undefined"
+      else
+        "simple(#{value})"
+      end
+    end
+    alias_method :inspect, :to_s
+  end
+
+  def self.encode(d)
+    new.add(d).buffer
+  end
+  def self.decode(s)
+    new(s).decode_item_final
+  end
+
+  attr_reader :buffer
+  def initialize(s = String.new)
+    @buffer = s
+    @pos = 0
+  end
+
+  def head(ib, n)
+    @buffer <<
+      case n
+      when 0...24
+        [ib + n].pack("C")
+      when 0...256
+        [ib + 24, n].pack("CC")
+      when 0...65536
+        [ib + 25, n].pack("Cn")
+      when 0...4294967296
+        [ib + 26, n].pack("CN")
+      when 0...18446744073709551616
+        [ib + 27, n].pack("CQ>")
+      else
+        yield                   # throw back to caller
+      end
+  end
+
+  HALF_NAN_BYTES = ("\xf9".force_encoding(Encoding::BINARY) + Half::NAN_BYTES).freeze
+
+  def addfloat(fv)
+    if fv.nan?
+      @buffer << HALF_NAN_BYTES
+    else
+      ss = [fv].pack("g")         # single-precision
+      if ss.unpack("g").first == fv
+        if hs = Half.encode_from_single(fv, ss)
+          @buffer << 0xf9 << hs
+        else
+          @buffer << 0xfa << ss
+        end
+      else
+        @buffer << [0xfb, fv].pack("CG") # double-precision
+      end
+    end
+  end
+
+  def bignum_to_bytes(d)
+    s = String.new
+    while (d != 0)
+      s << (d & 0xFF)
+      d >>= 8
+    end
+    s.reverse!
+  end
+
+  def add(d)
+    case d
+    when Integer
+      ib = if d < 0
+             d = -1-d
+             0x20
+           else
+             0x00
+           end
+      head(ib, d) {             # block is called if things do not fit
+        s = bignum_to_bytes(d)
+        head(0xc0, TAG_BIGNUM_BASE + (ib >> 5))
+        head(0x40, s.bytesize)
+        s
+      }
+    when Numeric; addfloat(d)
+    when Symbol; add(d.to_s)    # hack: this should really be tagged
+    when Simple; head(0xe0, d.value)
+    when false; head(0xe0, 20)
+    when true; head(0xe0, 21)
+    when nil; head(0xe0, 22)
+    when Tagged                 # we don't handle :simple here
+      head(0xc0, d.tag)
+      add(d.data)
+    when String
+      lengths = d.cbor_stream?
+      e = d
+      ib = if d.encoding == Encoding::BINARY
+             0x40
+           else
+             d = d.encode(Encoding::UTF_8).force_encoding(Encoding::BINARY)
+             0x60
+           end
+      if lengths
+        @buffer << (ib + 31)
+        pos = 0
+        lengths.each do |r|
+          add(e[pos, r])
+          pos += r
+        end
+        @buffer << 0xff
+      else
+        head(ib, d.bytesize)
+        @buffer << d
+      end
+    when Array
+      if d.cbor_stream?
+        @buffer << 0x9f
+        d.each {|di| add(di)}
+        @buffer << 0xff
+      else
+        head(0x80, d.size)
+        d.each {|di| add(di)}
+      end
+    when Hash
+      if d.cbor_stream?
+        @buffer << 0xbf
+        d.each {|k, v| add(k); add(v)}
+        @buffer << 0xff
+      else
+        head(0xa0, d.size)
+        d.each {|k, v| add(k); add(v)}
+      end
+    else
+      raise("Don't know how to encode #{d.inspect}")
+    end
+    self
+  end
+
+  def take(n)
+    opos = @pos
+    @pos += n
+    raise "Out of bytes to decode: #{opos} + #{n} > #{@buffer.bytesize}" if @pos > @buffer.bytesize
+    @buffer[opos, n]
+  end
+
+  MT_TO_ENCODING = {2 => Encoding::BINARY, 3 => Encoding::UTF_8}
+
+  def decode_item_streaming(ib, breakable)
+    case ib >>= 5
+    when 2, 3
+      want_encoding = MT_TO_ENCODING[ib]
+      subs = []
+      while (element = decode_item(true)) != BREAK
+        raise "non-string (#{element.inspect}) in streaming string" unless String === element
+        raise "bytes/text mismatch (#{element.encoding} != #{want_encoding}) in streaming string" unless element.encoding == want_encoding
+        subs << element
+      end
+      result = subs.join.cbor_stream!(subs.map(&:length)).force_encoding(want_encoding)
+    when 4
+      result = Array.new;
+      while (element = decode_item(true)) != BREAK
+        result << element
+      end
+      result
+    when 5
+      result = Hash.new
+      while (key = decode_item(true)) != BREAK
+        result[key] = decode_item
+      end
+      result
+    when 7
+      raise "break stop code outside indefinite length item" unless breakable
+      BREAK
+    else raise "unknown ib #{ib} for additional information 31"
+    end
+  end
+  
+  def decode_item(breakable = false)
+    ib = take(1).ord
+    ai = ib & 0x1F
+    val = case ai
+          when 0...24; ai
+          when 24; take(1).ord
+          when 25; take(2).unpack("n").first
+          when 26; (s = take(4)).unpack("N").first
+          when 27; (s = take(8)).unpack("Q>").first
+          when 31; return decode_item_streaming(ib, breakable)
+          else raise "unknown additional information #{ai} in ib #{ib}"
+          end
+    case ib >>= 5
+    when 0; val
+    when 1; -1-val
+    when 7
+      case ai
+      when 20; false
+      when 21; true
+      when 22; nil
+      # when 27; Simple.new(27)   # Ruby does not have Undefined
+      when 25; Half.decode(val)
+      when 26; s.unpack("g").first # cannot go directly from val
+      when 27; s.unpack("G").first #   in Ruby
+      else
+        Simple.new(val)
+      end
+    when 6
+      di = decode_item
+      if String === di && (val & ~1) == TAG_BIGNUM_BASE
+        (TAG_BIGNUM_BASE - val) ^ di.bytes.inject(0) {|sum, b| sum <<= 8; sum += b }
+      else
+        Tagged.new(val, di)
+      end
+    when 2; take(val).force_encoding(Encoding::BINARY)
+    when 3; take(val).force_encoding(Encoding::UTF_8)
+    when 4; Array.new(val) { decode_item }
+    when 5; Hash[Array.new(val) {[decode_item, decode_item]}]
+    end
+  end
+
+  def decode_item_final
+    val = decode_item
+    raise "extra bytes follow after a deserialized object" if @pos != @buffer.size
+    val
+  end
+
+end

data/lib/half.rb

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+#
+# 16-bit floating point values (IEEE 754 Half Precision) are not
+# supported by #pack/#unpack in Ruby yet.
+# This is a quick hack implementing en- and decoding them.
+# (Since this is just a hack, the brief tests are in this file.)
+#
+# The encoder assumes that we already have a Single-Precision byte
+# string (e.g., from pack("g")), and this is taken apart and
+# reassembled.
+# The decoder is free-standing (trivial).
+#
+# IEEE 754 can be found at:
+# http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=4610935
+
+module Half
+  NAN_BYTES = "\x7e\x00"
+
+  def self.decode_from_bytes(hs)
+    b16, = hs.unpack("n")
+    self.decode(b16)
+  end
+  def self.decode(b16)
+    exp = b16 >> 10 & 0x1f
+    mant = b16 & 0x3ff
+    val =
+      if exp == 0
+        Math.ldexp(mant, -24)
+      elsif exp == 31
+        mant == 0 ? Float::INFINITY : Float::NAN
+      else
+        Math.ldexp(0x400 + mant, exp-25)
+      end
+    if b16[15] != 0
+      -val
+    else
+      val
+    end
+  end
+
+  def self.encode_from_single_bytes(ss)        # single-precision string
+    b32, = ss.unpack("N")
+    s16 = b32 >> 16 & 0x8000
+    mant = b32 & 0x7fffff
+    exp = b32 >> 23 & 0xff
+    # puts "#{fv} #{s16} #{mant.to_s(16)} #{exp}"
+    if exp == 0
+      s16 if mant == 0            # 0.0, -0.0
+    elsif exp >= 103 && exp < 113 # denorm, exp16 = 0
+      s16 + ((mant + 0x800000) >> (126 - exp))
+    elsif exp >= 113 && exp <= 142 # normalized
+      s16 + ((exp - 112) << 10) + (mant >> 13)
+    elsif exp == 255              # Inf (handle NaN elsewhere!)
+      s16 + 0x7c00 if mant == 0   # +Inf/-Inf
+    end
+  end
+
+  def self.encode_from_single(fv, ss)
+    if e = self.encode_from_single_bytes(ss)
+      # p e.to_s(16)
+      hs = [e].pack("n")
+      hs if self.decode_from_bytes(hs) == fv
+    end
+  end
+
+  def self.encode(fv)
+    self.encode_from_single(fv, [fv].pack("g"))
+  end
+  
+end
+
+(-24..15).each do |i|
+  f = Math.ldexp(1, i)
+  s = Half.encode(f)
+  fail i unless s
+end
+(-24..6).each do |i|
+  f = Math.ldexp(1023, i)
+  s = Half.encode(f)
+  fail i unless s
+end
+
+# p Half.decode("\x7b\xff") 65504.0
+
+[                               # go through Wikipedia samples
+0b0_01111_0000000000, 1.0,
+0b0_01111_0000000001, 1.0 + Math.ldexp(1, -10), # = 1 + 2−10 = 1.0009765625 (next biggest float after 1)
+0b1_10000_0000000000, -2.0,
+
+0b0_11110_1111111111, 65504.0, #  (max half precision)
+
+0b0_00001_0000000000, Math.ldexp(1, -14), # ≈ 6.10352 × 10−5 (minimum positive normal)
+0b0_00000_1111111111, Math.ldexp(1, -14) - Math.ldexp(1, -24), # ≈ 6.09756 × 10−5 (maximum subnormal)
+0b0_00000_0000000001, Math.ldexp(1, -24), # ≈ 5.96046 × 10−8 (minimum positive subnormal)
+
+0b0_00000_0000000000, 0.0,
+0b1_00000_0000000000, -0.0,
+
+0b0_11111_0000000000, 1.0/0.0,
+0b1_11111_0000000000, -1.0/0.0,
+
+0b0_01101_0101010101, 0.333251953125 #... ≈ 1/3 
+].each_slice(2) do |hv, expected|
+  fv = Half.decode(hv)
+  raise [hv, fv, expected].inspect unless fv == expected
+end
+
+# NaN cannot be compared, so this one needs to be special-cased:
+raise "NaN not detected" unless Half.decode(0b0_11111_1000000000).nan?
+raise "-NaN not detected" unless Half.decode(0b1_11111_1000000000).nan?
+raise "NaN not detected" unless Half.decode_from_bytes(Half::NAN_BYTES).nan?