cbor-packed 0.1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7ec7bd11d45d35c418a0fe98aa43c369b3384b2b43cb94486fd6e3463eae3232
+  data.tar.gz: 88efc5decfeeedb1c049df0e5a9ba126c35bdf8a661218e29db3eb836c39f9c6
+SHA512:
+  metadata.gz: 1b6add7d575ff6ab3f239c9cf8f4f434ec8cc3c7d02e717c58b8c79a2007dd6d033ec10739c8bbac788e73d292f37501a002730303a8f58479a7131243fd04ac
+  data.tar.gz: efd9b83c4da71f11354398048e6421770583b2fef0e22b4e3ac1f964ba2d5f4d2d69b2cb1e6441058062313940d6fee492f6dd7579dd0e25202734229a33058b

data/cbor-packed.gemspec

@@ -0,0 +1,17 @@
+Gem::Specification.new do |s|
+  s.name = "cbor-packed"
+  s.version = "0.1.3"
+  s.summary = "CBOR (Concise Binary Object Representation) packer"
+  s.description = %q{cbor-packed implements packed encoding for CBOR, RFC 7049 Section 3.9}
+  s.author = "Carsten Bormann"
+  s.email = "cabo@tzi.org"
+  s.license = "Apache-2.0"
+  s.homepage = "http://cbor.io/"
+  s.has_rdoc = false
+  s.test_files = Dir['test/**/*.rb']
+  s.files = Dir['lib/**/*.rb'] + %w(cbor-packed.gemspec) + Dir['bin/**/*.rb']
+  s.executables = Dir['bin/**/*.rb'].map {|x| File.basename(x)}
+  s.required_ruby_version = '>= 1.9.2'
+
+  s.require_paths = ["lib"]
+end

data/lib/cbor-packed.rb

@@ -0,0 +1,305 @@
+# -*- coding: utf-8 -*-
+
+require "cbor" unless defined? CBOR
+# require "pp"
+
+$compression_hack = 0
+
+module CBOR
+  PACKED_TAG = 51
+  REF_TAG = 6
+
+  class Packer
+    def self.from_item(item)
+      count = Hash.new(0)
+      item.cbor_visit do |o|
+        (count[o] += 1) == 1
+        # if the count gets > 1, we can stop visiting, so we return false in the block
+      end
+      # pp count
+      # count is now a Hash with all data items as keys and the number of times they occur as values
+
+      # choose those matches that are occurring > 1, make first rough estimate of saving
+      good_count = count.select {|k, v| v > 1}.map {|k, v| [k, v, l = k.to_cbor.length,
+                                                            (v-1)*(l-1)]}
+      # good_count is now an array of [k, v, length, savings] tuples
+
+      # select those that potentially have savings (> 0) and sort by best saving first
+      better_count = good_count.to_a.select {|a| a[3] > 0}.sort_by {|a| -a[3]}
+      # pp better_count
+
+      # now: take the best out???; re-visit that reducing by n; re-sort and filter???
+      # sort by descending number of references we'll get -- the higher reference counts go first
+      match_array = better_count.sort_by {|a| -a[1]}.map {|a| a[0]}
+      # pp match_array
+
+      # XXX the below needs to be done with arrays and (hard!) maps as well
+      # do this on the reverse to find common suffixes
+      # select all strings (ignoring reference counts) and sort them
+      strings = count.select {|k, v| String === k}.map(&:first).sort
+      if strings != []
+        string_common = strings[1..-1].zip(strings).map{ |y, x|
+          l = x.chars.zip(y.chars).take_while{|a, b| a == b}.length # should be bytes
+          [x, l]
+        } << [strings[-1], 0]
+        # string_common: list of strings/counts of number of /bytes/ matching with next
+        # pp string_common
+      end
+      translate = {}
+      prefixes = []
+      if string_common
+      prefix_stack = [[0, false]] # sentinel
+      pos = 0                     # mirror prefix_stack[-1][0]
+      tag_no = REF_TAG
+      string_common.each do |s, l|
+        if l > pos + 2 + $compression_hack
+          if t = prefix_stack[-1][1] # if we still have a prefix left
+            prefixes << CBOR::Tagged.new(t, s[pos...l])
+          else
+            prefixes << s[0...l]
+          end
+          prefix_stack << [l, tag_no]
+          pos = l
+          tag_no += 1
+          tag_no = 225 if tag_no == REF_TAG+1
+          tag_no = 28704 if tag_no == 256
+        end
+        if t = prefix_stack[-1][1] # if we still have a viable prefix left
+          translate[s] = CBOR::Tagged.new(t, s[pos..-1])
+        end
+        # pop the prefix stack
+        while l < pos
+          prefix_stack.pop
+          pos = prefix_stack[-1][0]
+        end
+        # pp prefix_stack
+        # pp pos
+      end
+        
+      end
+      # pp translate
+      # XXX test replacing match_array here
+      match_array = match_array.map do |v|
+        if r = translate[v]
+          # puts "*** replacing #{v.inspect} by #{r.inspect}"
+          r
+        else
+          v
+        end
+      end
+      # pp [:PREFIXES, prefixes]
+      # pp translate
+      new(match_array, prefixes, [], translate)
+    end
+    def initialize(match_array, prefix_array, suffix_array, translate)
+      @hit = translate
+      # XXX: make sure we don't overwrite the existing prefix compression values!
+      # (this should really be done downwards, ...) 16 x 1, 160 x 2, (512-48) x 3
+      match_array[0...16].each_with_index do |o, i|
+        @hit[o] = CBOR::Simple.new(i)
+      end
+      # if m = match_array[16...128]
+      #   m.each_with_index do |o, i|
+      #     @hit[o] = CBOR::Simple.new(i + 128)
+      #   end
+      # end
+      if m = match_array[16..-1]
+        m.each_with_index do |o, i|
+          @hit[o] = CBOR::Tagged.new(REF_TAG, (i >> 1) ^ -(i & 1))
+        end
+      end
+      # add one round of transitive matching
+      @hit.each do |k, v|
+        if r = @hit[v]
+          @hit[k] = r
+        end
+      end
+      # p @hit
+      @match_array = match_array
+      # @prefix = {} -- do that later
+      @prefix_array = prefix_array
+      @suffix_array = suffix_array
+    end
+    def has(o)
+      @hit[o]
+    end
+    def pack(pa)
+      # Don't forget to pack the match_array!
+      CBOR::Tagged.new(PACKED_TAG, [@match_array, @prefix_array, @suffix_array, pa])
+    end
+  end
+
+  class Unpacker
+    def initialize(match_array, prefix_array, suffix_array)
+      @simple_array = match_array[0...16]
+      @tagged_array = match_array[16..-1]
+      # index with 2i for i >= 0 or ~2i for i < 0
+      # no map as we need to populate in progress
+      # pp prefix_array
+      @prefix_array = []
+      prefix_array.each {|x| @prefix_array << x.to_unpacked_cbor1(self)}
+      @suffix_array = []
+      suffix_array.each {|x| @prefix_array << x.to_unpacked_cbor1(self)}
+      # XXX order? -- must do lazily!
+    end
+    def unsimple(sv)
+      @simple_array[sv]
+    end
+    def untag(tv)
+      @tagged_array[(i << 1) ^ (i >> 63)]
+    end
+    def unprefix(n)
+      @prefix_array[n]
+    end
+    def unsuffix(n)
+      @suffix_array[n]
+    end
+  end
+
+  module Packed
+
+    module Object_Packed_CBOR
+      def cbor_visit
+        yield self
+      end
+      def to_unpacked_cbor1(unpacker)
+        self
+      end
+      def to_packed_cbor(packer = Packer.from_item(self))
+        packer.pack(to_packed_cbor1(packer))
+      end
+      def to_packed_cbor1(packer = Packer.from_item(self))
+        if c = packer.has(self)
+          c
+        else
+          # Need to do the prefix dance, too
+          self
+        end
+      end
+    end
+    Object.send(:include, Object_Packed_CBOR)
+
+    module Simple_Packed_CBOR
+      def to_unpacked_cbor1(unpacker)
+        if v = unpacker.unsimple(value)
+          v.to_unpacked_cbor1(unpacker)
+        else
+          self
+        end
+      end
+    end
+    CBOR::Simple.send(:include, Simple_Packed_CBOR)
+
+    module String_Packed_CBOR
+      def packed_merge(other, unpacker)
+        # add checks
+        to_unpacked_cbor1(unpacker) + other.to_unpacked_cbor1(unpacker)
+      end
+    end
+    String.send(:include, String_Packed_CBOR)
+
+    module Array_Packed_CBOR
+      def cbor_visit(&b)
+        if yield self
+          each do |o|
+            o.cbor_visit(&b)
+          end
+        end
+      end
+      def to_unpacked_cbor1(unpacker)
+        map {|x| x.to_unpacked_cbor1(unpacker)}
+      end
+      def to_packed_cbor1(packer = Packer.from_item(self))
+        if c = packer.has(self)
+          c.to_unpacked_cbor1(unpacker)
+        else
+          # TODO: Find useful prefixes
+          map {|x| x.to_packed_cbor1(packer)}
+        end
+      end
+      def packed_merge(other, unpacker)
+        # TODO: add checks
+        to_unpacked_cbor1(unpacker) + other.to_unpacked_cbor1(unpacker)
+      end
+    end
+    Array.send(:include, Array_Packed_CBOR)
+
+    module Hash_Packed_CBOR
+      def cbor_visit(&b)
+        if yield self
+          each do |k, v|
+            k.cbor_visit(&b)
+            v.cbor_visit(&b)
+          end
+        end
+      end
+      def to_unpacked_cbor1(unpacker)
+        Hash[map {|k, v| [k.to_unpacked_cbor1(unpacker), v.to_unpacked_cbor1(unpacker)]}]
+      end
+      def to_packed_cbor1(packer = Packer.from_item(self))
+        if c = packer.has(self)
+          c.to_unpacked_cbor1(unpacker)
+        else
+          # TODO: Find useful prefixes
+          Hash[map {|k, v| [k.to_packed_cbor1(packer), v.to_packed_cbor1(packer)]}]
+        end
+      end
+      def packed_merge(other, unpacker)
+        # TODO: add checks
+        to_unpacked_cbor1(unpacker).merge other.to_unpacked_cbor1(unpacker)
+      end
+    end
+    Hash.send(:include, Hash_Packed_CBOR)
+
+    module Tagged_Packed_CBOR
+      def cbor_visit(&b)
+        if yield self
+          value.cbor_visit(&b)
+        end
+      end
+      def to_unpacked_cbor
+        if tag == PACKED_TAG
+          # check that this really is an array
+          # warn value.to_yaml
+          ma, pa, sa, pv = value
+          unpacker = Unpacker.new(ma, pa, sa)
+          pv.to_unpacked_cbor1(unpacker)
+        else
+          fail "error message here"
+        end
+      end
+      def to_unpacked_cbor1(unpacker)
+        case tag
+        when REF_TAG
+          if Integer === value
+            unpacker(untag(value))
+          else
+            unpacker.unprefix(0).packed_merge value, unpacker
+          end
+        when 225...256
+          unpacker.unprefix(tag-(256-32)).packed_merge value, unpacker
+        when 28704...32768      # (- 28704 (- 32768 4096)) == 32
+          unpacker.unprefix(tag-(32768-4096)).packed_merge value, unpacker
+        when 1879052288...2147483648 # (- 1879052288 (- 2147483648 268435456)) == 4096
+          unpacker.unprefix(tag-(2147483648-268435456)).packed_merge value, unpacker
+        when 216...224
+          value.packed_merge unpacker.unsuffix(tag-216), unpacker
+        when 27647...28672
+          value.packed_merge unpacker.unsuffix(tag-(28672-1024)), unpacker
+        when 1811940352...1879048192
+          value.packed_merge unpacker.unsuffix(tag-(1879048192-67108864)), unpacker
+        else
+          CBOR::Tagged.new(tag, value.to_unpacked_cbor1(unpacker))
+        end
+      end
+      def to_packed_cbor1(packer = Packer.from_item(self))
+        if c = packer.has(self)
+          c
+        else
+          CBOR::Tagged.new(tag, value.to_packed_cbor1(packer))
+        end
+      end
+    end
+    CBOR::Tagged.send(:include, Tagged_Packed_CBOR)
+  end
+end

data/test/exa2.rb

@@ -0,0 +1,37 @@
+require 'cbor-pure'
+require 'cbor-packed'
+require 'cbor-diagnostic'
+require 'json'
+require 'yaml'
+
+fn = "/Users/cabo/big/wot-thing-description/test-bed/data/plugfest/2017-05-osaka/MyLED_f.jsonld"
+puts fn.split("/")[4..-1].join("/")
+jf = File.read(fn)
+puts "JSON file: #{jf.length}"
+jo = JSON.parse(jf)
+jsw = JSON::generate(jo, :allow_nan => true, :max_nesting => false)
+# l3 jsw, "JSON no whitespace"
+File.write("/tmp/jsw", jsw)
+
+# p jo
+cb = jo.to_cbor
+# l3 cb, "CBOR"
+$compression_hack = 1000
+pa = jo.to_packed_cbor
+cbp = pa.to_cbor
+# l3 cbp, "CBOR packed"
+puts "CBOR packed (sharing only): #{cbp.length}"
+$compression_hack = 0
+pa = jo.to_packed_cbor
+cbp = pa.to_cbor
+# l3 cbp, "CBOR packed"
+puts "CBOR packed: #{cbp.length}"
+File.write("/tmp/cbp", cbp)
+File.write("/tmp/cbu", cb)
+pad = CBOR.decode(cbp)
+up =  pad.to_unpacked_cbor
+pp up == jo
+if up != jo
+  puts up.to_yaml
+  puts pad.to_yaml   # get rid of unwanted sharing...
+end

data/test/exa2r.rb

@@ -0,0 +1,59 @@
+require 'cbor-pure'
+require 'cbor-packed'
+require 'treetop'
+require 'cbor-diag-parser'
+require 'cbor-diagnostic'
+require 'json'
+require 'yaml'
+
+input = DATA.read
+
+
+parser = CBOR_DIAGParser.new
+if result = parser.parse(input)
+  o = result.to_rb.to_unpacked_cbor
+  puts o.cbor_diagnostic
+else
+  puts "*** can't parse #{i}"
+  puts "*** #{parser.failure_reason}"
+end
+
+# ruby -I ../lib exa2r.rb | jq > .out1
+# jq < ~/big/wot-thing-description/test-bed/data/plugfest/2017-05-osaka/MyLED_f.jsonld > .out2
+# ruby .check
+# => true
+
+
+__END__
+
+ 51([/shared/["name", "@type", "links", "href", "mediaType",
+             /  0       1       2        3         4 /
+     "application/json", "outputData", {"valueType": {"type":
+          /  5               6               7 /
+     "number"}}, ["Property"], "writable", "valueType", "type"],
+                /   8            9           10           11 /
+    /prefix/ ["http://192.168.1.10", 6("3:8445/wot/thing"),
+               / 6                        225 /
+    225("/MyLED/"), 226("rgbValue"), "rgbValue",
+      / 226             227           228     /
+    {simple(6): simple(7), simple(9): true, simple(1): simple(8)}],
+      / 229 /
+    /suffix/ [],
+    /rump/ {simple(0): "MyLED",
+            "interactions": [
+      229({simple(2): [{simple(3): 227("Red"), simple(4): simple(5)}],
+       simple(0): 228("Red")}),
+      229({simple(2): [{simple(3): 227("Green"), simple(4): simple(5)}],
+       simple(0): 228("Green")}),
+      229({simple(2): [{simple(3): 227("Blue"), simple(4): simple(5)}],
+       simple(0): 228("Blue")}),
+      229({simple(2): [{simple(3): 227("White"), simple(4): simple(5)}],
+       simple(0): "rgbValueWhite"}),
+      {simple(2): [{simple(3): 226("ledOnOff"), simple(4): simple(5)}],
+       simple(6): {simple(10): {simple(11): "boolean"}}, simple(0):
+       "ledOnOff", simple(9): true, simple(1): simple(8)},
+      {simple(2): [{simple(3): 226("colorTemperatureChanged"),
+       simple(4): simple(5)}], simple(6): simple(7), simple(0):
+       "colorTemperatureChanged", simple(1): ["Event"]}],
+      simple(1): "Lamp", "id": "0", "base": 225(""),
+      "@context": 6("2:8444/wot/w3c-wot-td-context.jsonld")}])

data/test/test-packed.rb

@@ -0,0 +1,101 @@
+require 'cbor-pure'
+require 'cbor-packed'
+require 'cbor-diagnostic'
+require 'json'
+require 'yaml'
+require 'zlib'
+require 'lz4-ruby'
+
+def l3(bin, desc)
+  puts "#{desc}: #{bin.length}, deflate: #{Zlib.deflate(bin, Zlib::BEST_COMPRESSION).length}, lz4: #{LZ4::compress(bin).length}, lz4hc: #{LZ4::compressHC(bin).length}"
+end
+
+fn = "/Users/cabo/big/wot-thing-description/test-bed/data/plugfest/2017-05-osaka/MyLED_f.jsonld"
+puts fn.split("/")[4..-1].join("/")
+jf = File.read(fn)
+puts "JSON file: #{jf.length}"
+jo = JSON.parse(jf)
+jsw = JSON::generate(jo, :allow_nan => true, :max_nesting => false)
+l3 jsw, "JSON no whitespace"
+File.write("/tmp/jsw", jsw)
+
+# p jo
+cb = jo.to_cbor
+l3 cb, "CBOR"
+$compression_hack = 1000
+pa = jo.to_packed_cbor
+cbp = pa.to_cbor
+# l3 cbp, "CBOR packed"
+puts "CBOR packed (sharing only): #{cbp.length}"
+$compression_hack = 0
+pa = jo.to_packed_cbor
+cbp = pa.to_cbor
+# l3 cbp, "CBOR packed"
+puts "CBOR packed: #{cbp.length}"
+File.write("/tmp/cbp", cbp)
+File.write("/tmp/cbu", cb)
+pad = CBOR.decode(cbp)
+# puts pad.to_yaml
+up =  pad.to_unpacked_cbor
+pp up == jo
+if up != jo
+  puts up.to_yaml
+  puts pad.to_yaml   # get rid of unwanted sharing...
+end
+
+# puts "#{pa.value.map {|x| x.to_cbor.length}}"
+puts pa.value[1].to_json
+
+exin = JSON.load(DATA)
+exout = exin.to_packed_cbor
+p CBOR.encode(exin).length, CBOR.encode(exout).length
+puts exout.cbor_diagnostic
+if exin != exout.to_unpacked_cbor
+  fail exout.inspect
+end
+
+exit
+
+# $compression_hack = 1000
+
+a1 = {"aaaa" => "aaaaaaaa", "aaaaaaaaaaaa" => "aaaa", "aaaaaaaa" => "aaaaaaaaaaaa"}
+pa1 = a1.to_packed_cbor
+puts pa1.to_yaml
+a1u = pa1.to_unpacked_cbor
+if a1u != a1
+  puts a1u.to_yaml
+end
+
+__END__
+
+{ "store": {
+    "book": [ 
+      { "category": "reference",
+        "author": "Nigel Rees",
+        "title": "Sayings of the Century",
+        "price": 8.95
+      },
+      { "category": "fiction",
+        "author": "Evelyn Waugh",
+        "title": "Sword of Honour",
+        "price": 12.99
+      },
+      { "category": "fiction",
+        "author": "Herman Melville",
+        "title": "Moby Dick",
+        "isbn": "0-553-21311-3",
+        "price": 8.95
+      },
+      { "category": "fiction",
+        "author": "J. R. R. Tolkien",
+        "title": "The Lord of the Rings",
+        "isbn": "0-395-19395-8",
+        "price": 22.99
+      }
+    ],
+    "bicycle": {
+      "color": "red",
+      "price": 19.95
+    }
+  }
+}

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: cbor-packed
+version: !ruby/object:Gem::Version
+  version: 0.1.3
+platform: ruby
+authors:
+- Carsten Bormann
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-05-04 00:00:00.000000000 Z
+dependencies: []
+description: cbor-packed implements packed encoding for CBOR, RFC 7049 Section 3.9
+email: cabo@tzi.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- cbor-packed.gemspec
+- lib/cbor-packed.rb
+- test/exa2.rb
+- test/exa2r.rb
+- test/test-packed.rb
+homepage: http://cbor.io/
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.15
+signing_key:
+specification_version: 4
+summary: CBOR (Concise Binary Object Representation) packer
+test_files:
+- test/exa2.rb
+- test/exa2r.rb
+- test/test-packed.rb