tuplex 0.1 → 0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b58ffe6b6580580163d4fe58df687ad5ea20e6de
-  data.tar.gz: 5289b5307ea0788d2ad79ba50f8ee9b957f5b714
+  metadata.gz: 0afb0b081f7c4846c6aa138ea975f6ed30ab7e04
+  data.tar.gz: d6c419f04e8c802334eb27d3a1e9fb9f0325951d
 SHA512:
-  metadata.gz: 652c10a64977a7b1a61b908b438615e47bd6f74c7db78e141bb6cb84552d51357dfb0078080cd8146fbbb048a95f7ff3ccbe6125d8f96761df2df4a9c550ff6d
-  data.tar.gz: 7edc4d53d20e441c2feb42957e46e91de37e30aea649024743e59385e4b64df3723f1211c98a7e4b423d41aa73c9335401d72ad2727a5be4be293395ab799a40
+  metadata.gz: 25d656204c50d81c77fef4a49a570ada7c3960c8d1d441c879de35329e4151fa01a28d90a4a5f51e86cb98b187881efeab8001f0f943151b6334f405dcb64aaf
+  data.tar.gz: c290e67ad062ea39e214881a19b1385f6d36b802d013be527201d2c2261356f18b3212b36877ff217e16b57bac4433f3bd333914503fd9d6f127601b82aaf91a

data/README.md

@@ -3,6 +3,57 @@ Tuplex
 
 Constructs index keys for tuples.
 
+What is it for?
+---
+
+1. Assign keys to tuples (immutable value types) that do not have unique ids (primary keys). This can be used to store a tuplespace in a key-value store.
+
+2. Keys for similar tuples should be similar, to keep them close in storage, reducing cache misses and page faults.
+
+3. The key construction preserves ordering in such a way that the index can be used as an approximate secondary index for multidimensional range queries.
+
+What does it do?
+---
+
+Tuplex gives you a function that turns a tuple into a string that can be used as a key in a key-value store. The key is non-unique, so you'll need to use the value to disambiguate (see examples).
+
+For tuples of a given _signature_ (same array sizes, map keys, type of each value, etc.), the function is _monotonic_ on each value. For example:
+
+    ["foo", 1, 2]
+    ["foo", 1, 3]
+
+These two tuples have the same signature: three elements of types string, number, and number, respectively.
+
+The index keys for these tuples are as follows:
+
+    >> Tuplex.make_key(["foo", 1, 2])
+    => "\x97\xB0kL\xA0\xC9\x00\xD1\x00hwn"
+    >> Tuplex.make_key(["foo", 1, 3])
+    => "\x97\xB0kL\xA0\xC9\x00\xD1\x00hwn\x00\b"
+    >> Tuplex.make_key(["foo", 1, 2]) < Tuplex.make_key(["foo", 1, 3])
+    => true
+
+So, the ordering `2<3` is preserved in the key strings (lexically ordered).
+
+This is also true when varying any number of terms, whether string or number:
+
+    >> Tuplex.make_key(["foo", 1, 2]) < Tuplex.make_key(["foozap", 7, 3])
+    => true
+
+And it's true for arbitrary nesting:
+
+    >> Tuplex.make_key(["foo", {a: 1, b: [2]}]) < Tuplex.make_key(["foozap", {a: 7, b: [3]}])
+    => true
+
+However, for tuples of different signatures, the ordering depends only on the signature and not on term values:
+
+    >> Tuplex.make_key(["a", 0]) < Tuplex.make_key([0, "a"])
+    => true
+    >> Tuplex.make_key(["z", 1000]) < Tuplex.make_key([0, "a"])
+    => true
+
+In other words, all tuples of signature (String, Number) are contiguous in the index, and that contiguous group is separate from tuples of signature (Number, String).
+
 Contact
 =======
 

data/examples/lmdb-sorted.rb

@@ -0,0 +1,31 @@
+require 'tuplex'
+require 'lmdb'
+require 'tmpdir'
+
+include Tuplex
+
+dir = Dir.mktmpdir
+env = LMDB.new dir
+db = @db = env.database("tuples", create: true, dupsort: true)
+
+def store t
+  @db[make_key(t)] = make_val(t)
+end
+
+(1..10).to_a.shuffle.each do |i|
+  store [1, 2, i]       # same signature for each i
+  store a: "foo", b: i  # same signature for each i, but different from above
+  store i => nil        # different signature for each i
+end
+
+puts "\n\nIterating..."
+db.each do |k,v|
+  p unpack_val(v)
+end
+
+puts "\n\nSearching..."
+db.each do |k,v|
+  if val_equals_tuple(v, [1, 2, 5])
+    puts "found!"
+  end
+end

data/examples/lmdb.rb

@@ -0,0 +1,14 @@
+require 'tuplex'
+require 'lmdb'
+require 'tmpdir'
+
+include Tuplex
+
+dir = Dir.mktmpdir
+env = LMDB.new dir
+db = env.database("tuples", create: true, dupsort: true)
+
+t = {a: 1, b: 2}
+db[make_key(t)] = make_val(t)
+
+p db.map {|k,v| [k, unpack_val(v)]}

data/lib/tuplex.rb

@@ -0,0 +1,98 @@
+require 'msgpack'
+
+module Tuplex
+  module_function
+
+  # +t+ can be a tuple or a value in a tuple (that is, an entry in
+  # an array or a value at some key in a hash).
+  def signature t
+    case t
+      when nil, true, false; 0
+      when Numeric; 1
+      when String, Symbol; 2
+      when Array
+        t.map {|v| signature(v)}
+      when Hash
+        t.each_with_object({}) {|(k,v), h| h[k] = signature(v)}
+      else raise ArgumentError, "cannot compute signature for #{t.inspect}"
+    end
+  end
+
+  SIG_KEY_SIZE = 8
+  def sig_key t
+    [signature(t).hash].pack("q")
+  end
+
+  def str_sum acc, s
+    a = acc.unpack("C*")
+    b = s.unpack("C*")
+    if a.size < b.size
+      a,b = b,a
+    end
+    cv = 0
+    s = []
+    a.zip(b).reverse_each do |av,bv|
+      bv ||= 0
+      cv,r = (av + bv + cv).divmod 256
+      s << r
+    end
+    if cv != 0
+      raise "overflow"
+    end
+    s.reverse.pack("C*")
+  end
+
+  # https://en.wikipedia.org/wiki/Double-precision_floating-point_format
+  # "%064b" % [-1.0].pack("G").unpack("Q>")
+  # [0b1011111111110000000000000000000000000000000000000000000000000000].pack("Q>").unpack("G")
+
+  def expo(x) ([x].pack("G")[0..1].unpack("S>")[0] & 0b0111111111110000) >> 4; end
+  def mant(x) [x].pack("G").unpack("Q>")[0] & 0x000FFFFFFFFFFFFF; end
+
+  def float_to_key x
+    if x >= 0
+      [1, expo(x), mant(x)].pack("CS>Q>") # sparse
+    else
+      [0, -expo(x), -mant(x)].pack("CS>Q>")
+    end
+  end
+  # def fk(x); "%064b" % float_to_key(x).unpack("Q>"); end
+
+  MAX_SUM_KEY_SIZE = 500
+  def sum_key t, acc = "\0\0"
+    case t
+    when nil;   str_sum(acc, "\0\0")
+    when false; str_sum(acc, "\0\1")
+    when true;  str_sum(acc, "\0\2")
+    when Numeric; str_sum(acc, "\0" + float_to_key(t.to_f))
+    when String; str_sum(acc, "\0" + t) # truncate here
+    when Symbol; str_sum(acc, "\0" + t.to_s) # and here
+    when Array; t.inject(acc) {|s,v| sum_key(v,s)}
+    when Hash; t.inject(acc) {|s,(k,v)| sum_key(v,s)}
+    else raise ArgumentError, "bad type: #{t.inspect}"
+    end
+  end
+
+  MAX_KEY_SIZE = SIG_KEY_SIZE + MAX_SUM_KEY_SIZE
+  # note: MDB_MAXKEYSIZE is 511
+
+  def make_key t
+    (sig_key(t) + sum_key(t))[0..MAX_KEY_SIZE].sub(/\0+\z/, "")
+  end
+
+  def make_val t
+    make_val_hash(t) + MessagePack.pack(t)
+  end
+
+  def make_val_hash t
+    [t.hash].pack("Q>")
+  end
+
+  def unpack_val s
+    MessagePack.unpack(s[8..-1])
+  end
+
+  def val_equals_tuple s, t, th = make_val_hash(t)
+    s[0..7] == th && unpack_val(s) == t
+  end
+end

data/lib/tuplex/version.rb

@@ -1,3 +1,3 @@
 module Tuplex
-  VERSION = "0.1"
+  VERSION = "0.2"
 end

data/test/test-monotonic.rb

@@ -0,0 +1,54 @@
+require 'minitest/autorun'
+require 'tuplex'
+
+class TestMonotonic < Minitest::Test
+  include Tuplex
+
+  def make_key_pairs vals, frame = proc {|x| [x]}
+    vals.sort.map { |x|
+      t = frame[x]
+      [t, make_key(t)]
+    }
+  end
+
+  def assert_monotonic pairs
+    pairs.each_cons(2) {|(t1,s1),(t2,s2)|
+      assert_operator(s1, :<, s2, "comparing #{t1.inspect} < #{t2.inspect}")
+    }
+  end
+
+  FRAMES = [
+    proc {|x| [x]},
+    proc {|x| {a: "foo", b: 123.456, c: [1, 2, x, 3, 4], d: 1.23}}
+  ]
+
+  def test_numerics
+    ints = [-2**62, -123456, -10, -1, 0, 1, 2, 11, 99999, 2**62]
+    floats = [
+      -1.0/0, -1.23e300, -4.56e10, -7.89e-200,
+      1.23e-200, 4.56e10, 7.89e300, 1.0/0]
+
+    FRAMES.each do |frame|
+      assert_monotonic(make_key_pairs(ints+floats, frame))
+    end
+
+    assert_equal(make_key(0), make_key(0.0))
+  end
+
+  def test_strings
+    strs = ["", "a", "aa", "ab", "b", "bb"]
+
+    FRAMES.each do |frame|
+      assert_monotonic(make_key_pairs(strs, frame))
+    end
+
+    assert_equal(make_key(""), make_key("\0"))
+  end
+
+  def test_overflow
+    t = ["\xFF"] * 0x102
+    assert_raises(RuntimeError) {
+      make_key(t)
+    }
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tuplex
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: '0.2'
 platform: ruby
 authors:
 - Joel VanderWerf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-01 00:00:00.000000000 Z
+date: 2014-09-04 00:00:00.000000000 Z
 dependencies: []
 description: Constructs index keys for tuples.
 email: vjoel@users.sourceforge.net
@@ -21,13 +21,17 @@ extra_rdoc_files:
 files:
 - COPYING
 - README.md
+- examples/lmdb-sorted.rb
+- examples/lmdb.rb
 - ext/isaac/extconf.rb
 - ext/isaac/isaac.c
 - ext/isaac/rand.c
 - ext/isaac/rand.h
 - ext/isaac/rand4.c
 - ext/isaac/rand4.h
+- lib/tuplex.rb
 - lib/tuplex/version.rb
+- test/test-monotonic.rb
 homepage: https://github.com/vjoel/tuplex
 licenses: []
 metadata: {}
@@ -59,5 +63,6 @@ rubygems_version: 2.4.1
 signing_key: 
 specification_version: 4
 summary: Tuple index.
-test_files: []
+test_files:
+- test/test-monotonic.rb
 has_rdoc: