libcdb-ruby 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7a350d25e790bf8b2e405067e69a154393a0749e
-  data.tar.gz: 51c44fe36fd9ac0130e76a1c7d19de6ee4d8db94
+  metadata.gz: 16c0aecf11141408bf924eb8fb720274b00243ff
+  data.tar.gz: a18316f3428804a9e75b290343f14c58a874608f
 SHA512:
-  metadata.gz: 73d37a569b353eac9c087b96498f4974115679824a3014b6133e8c7288675797e882442d6fb07adf6378dc02b9fb9a1bb0c0058bc551cfe2adcb1e0a09ef7759
-  data.tar.gz: 452daa03315b780f5c2d222834982aedec3d552f942371b76c82bb3a4617f09af7a486220c19d251c08982b8371c48982652e31a023332d1e96e8bc6bba4864b
+  metadata.gz: a076b50bcebf0e90f5d889e14a8e615f41dd8287b2c1e699cfec742f7cf9e5596108f4bae23089551c1b583ab867976bfa993c552ce3f59f20445cf682564c8a
+  data.tar.gz: a776122e6ae6b44a1bba030cbb0c3073f0ea9e2636b4ebd39de9435c2c514291890bc1b4e693c4e79e97ab904c66e17da0e125d7a2386fe670c83d10f4989687

data/ChangeLog

@@ -2,6 +2,16 @@
 
 = Revision history for libcdb-ruby
 
+== 0.2.0 [2014-12-05]
+
+* Added encoding support to LibCDB::CDB::Reader.
+* Added LibCDB::CDB.load and LibCDB::CDB.load_file to create a database from a
+  dump.
+* Added LibCDB::CDB.stats and LibCDB::CDB.print_stats to collect stats from a
+  database.
+* Fixed that LibCDB::CDB::Reader#each_key and LibCDB::CDB::Reader#each_value
+  would not return an enumerator when no block was given.
+
 == 0.1.1 [2014-04-25]
 
 * <b>Dropped support for Ruby 1.9.2.</b>

data/README

@@ -2,7 +2,7 @@
 
 == VERSION
 
-This documentation refers to libcdb-ruby version 0.1.1
+This documentation refers to libcdb-ruby version 0.2.0
 
 
 == DESCRIPTION
@@ -74,7 +74,7 @@ creating and reading {constant databases}[http://cr.yp.to/cdb.html].
 == SUPPORTED PLATFORMS
 
 Linux::   MRI 1.9.3, 2.0 & 2.1 (Tested on 64-bit Ubuntu GNU/Linux
-          with 1.9.3p488, 2.0.0p377 and 2.1.2p77)
+          with 1.9.3p550, 2.0.0p594 and 2.1.5p273)
 Windows:: MRI 1.9.3 (Tested on 32-bit Windows XP with 1.9.3p194)
 
 
@@ -82,7 +82,7 @@ Windows:: MRI 1.9.3 (Tested on 32-bit Windows XP with 1.9.3p194)
 
 CDB::           http://cr.yp.to/cdb.html
 TinyCDB::       http://www.corpit.ru/mjt/tinycdb.html
-Documentation:: https://blackwinter.github.io/libcdb-ruby/
+Documentation:: https://blackwinter.github.com/libcdb-ruby
 Source code::   https://github.com/blackwinter/libcdb-ruby
 RubyGem::       https://rubygems.org/gems/libcdb-ruby
 Travis CI::     https://travis-ci.org/blackwinter/libcdb-ruby

data/Rakefile

@@ -1,4 +1,4 @@
-require File.expand_path(%q{../lib/libcdb/version}, __FILE__)
+require_relative 'lib/libcdb/version'
 
 begin
   require 'hen'

data/ext/libcdb/ruby_cdb.h

@@ -38,17 +38,23 @@ rcdb_##what##er_alloc(VALUE klass) {\
     rb_sys_fail(0);\
   }
 
-#define RCDB_RAISE_ARGS(min, max) \
+#define RCDB_RAISE_ARGS0(min, max, argc) \
   rb_raise(rb_eArgError,\
     "wrong number of arguments (%d for " #min "-" #max ")", argc);
 
-#define RCDB_RETURN_ENUMERATOR(self, argc, argv, max) \
+#define RCDB_RAISE_ARGS(min, max) RCDB_RAISE_ARGS0(min, max, argc)
+
+#define RCDB_RETURN_ENUMERATOR0(argc, argv, max) \
   if (argc > max) {\
-    RCDB_RAISE_ARGS(0, max)\
+    RCDB_RAISE_ARGS0(0, max, argc)\
   }\
 \
   RETURN_ENUMERATOR(self, argc, argv)
 
+#define RCDB_RETURN_ENUMERATOR(max) RCDB_RETURN_ENUMERATOR0(argc, argv, max)
+
+#define RCDB_RETURN_ENUMERATOR_NONE RCDB_RETURN_ENUMERATOR0(0, NULL, 0)
+
 #define RCDB_DEFINE_INSPECT(what) \
 static VALUE \
 rcdb_##what##er_inspect(VALUE self) {\

data/ext/libcdb/ruby_cdb_reader.c

@@ -25,6 +25,7 @@ rcdb_reader_closed_p(VALUE self) {
 static VALUE
 rcdb_reader_initialize(VALUE self, VALUE io) {
   RCDB_INITIALIZE(read, READ, cdb, init)
+  rb_iv_set(self, "@encoding", rb_enc_default_external());
   return self;
 }
 
@@ -40,22 +41,20 @@ rcdb_reader_iter_push(VALUE val, VALUE ary) {
 /* Helper method */
 static VALUE
 rcdb_reader_iter_aset(VALUE pair, VALUE hash) {
-  VALUE key = rb_ary_entry(pair, 0), val = rb_ary_entry(pair, 1), old;
-
-  if (!st_lookup(RHASH_TBL(hash), key, 0)) {
-    rb_hash_aset(hash, key, val);
-  }
-  else {
-    old = rb_hash_aref(hash, key);
-
-    switch (TYPE(old)) {
-      case T_ARRAY:
-        rb_ary_push(old, val);
-        break;
-      default:
-        rb_hash_aset(hash, key, rb_ary_new3(2, old, val));
-        break;
-    }
+  VALUE key = rb_ary_entry(pair, 0);
+  VALUE val = rb_ary_entry(pair, 1);
+  VALUE old = rb_hash_aref(hash, key);
+
+  switch (TYPE(old)) {
+    case T_NIL:
+      rb_hash_aset(hash, key, val);
+      break;
+    case T_ARRAY:
+      rb_ary_push(old, val);
+      break;
+    default:
+      rb_hash_aset(hash, key, rb_ary_new3(2, old, val));
+      break;
   }
 
   return Qnil;
@@ -104,9 +103,9 @@ rcdb_reader_dump_pair(VALUE key, VALUE val) {
   VALUE str = rb_str_new2("");
 
   rb_str_cat2(str, "+");
-  rb_str_append(str, rb_fix2str(LONG2NUM(RSTRING_LEN(key)), 10));
+  rb_str_append(str, RCDB_READER_STRING_LEN(key));
   rb_str_cat2(str, ",");
-  rb_str_append(str, rb_fix2str(LONG2NUM(RSTRING_LEN(val)), 10));
+  rb_str_append(str, RCDB_READER_STRING_LEN(val));
   rb_str_cat2(str, ":");
   rb_str_append(str, key);
   rb_str_cat2(str, "->");
@@ -144,7 +143,7 @@ rcdb_reader_each(int argc, VALUE *argv, VALUE self) {
   unsigned cdbp;
   VALUE key;
 
-  RCDB_RETURN_ENUMERATOR(self, argc, argv, 1);
+  RCDB_RETURN_ENUMERATOR(1);
   RCDB_READER_GET(self, cdb);
 
   if (rb_scan_args(argc, argv, "01", &key) == 1 && !NIL_P(key)) {
@@ -155,7 +154,7 @@ rcdb_reader_each(int argc, VALUE *argv, VALUE self) {
     }
 
     while (cdb_findnext(&cdbf) > 0) {
-      rb_yield(rcdb_reader_read_data(cdb));
+      rb_yield(RCDB_READER_READ(data));
     }
   }
   else {
@@ -163,8 +162,8 @@ rcdb_reader_each(int argc, VALUE *argv, VALUE self) {
 
     while (cdb_seqnext(&cdbp, cdb) > 0) {
       rb_yield(rb_ary_new3(2,
-        rcdb_reader_read_key(cdb),
-        rcdb_reader_read_data(cdb)));
+        RCDB_READER_READ(key),
+        RCDB_READER_READ(data)));
     }
   }
 
@@ -184,9 +183,11 @@ static VALUE
 rcdb_reader_each_dump(int argc, VALUE *argv, VALUE self) {
   VALUE key;
 
-  RCDB_RETURN_ENUMERATOR(self, argc, argv, 1);
+  RCDB_RETURN_ENUMERATOR(1);
 
   if (rb_scan_args(argc, argv, "01", &key) == 1 && !NIL_P(key)) {
+    StringValue(key);
+
     RCDB_READER_ITERATE0(each, yield_dump2, rb_ary_new3(1, key))
   }
   else {
@@ -210,11 +211,12 @@ rcdb_reader_each_key(VALUE self) {
   unsigned cdbp;
   VALUE key, hash = rb_hash_new();
 
+  RCDB_RETURN_ENUMERATOR_NONE;
   RCDB_READER_GET(self, cdb);
   cdb_seqinit(&cdbp, cdb);
 
   while (cdb_seqnext(&cdbp, cdb) > 0) {
-    if (!st_lookup(RHASH_TBL(hash), key = rcdb_reader_read_key(cdb), 0)) {
+    if (NIL_P(rb_hash_lookup(hash, key = RCDB_READER_READ(key)))) {
       rb_hash_aset(hash, key, Qtrue);
       rb_yield(key);
     }
@@ -236,11 +238,12 @@ rcdb_reader_each_value(VALUE self) {
   struct cdb *cdb = NULL;
   unsigned cdbp;
 
+  RCDB_RETURN_ENUMERATOR_NONE;
   RCDB_READER_GET(self, cdb);
   cdb_seqinit(&cdbp, cdb);
 
   while (cdb_seqnext(&cdbp, cdb) > 0) {
-    rb_yield(rcdb_reader_read_data(cdb));
+    rb_yield(RCDB_READER_READ(data));
   }
 
   return self;
@@ -274,7 +277,7 @@ rcdb_reader_fetch_first(VALUE self, VALUE key) {
   RCDB_READER_GET(self, cdb);
 
   if (cdb_find(cdb, RSTRING_PTR(key), RSTRING_LEN(key)) > 0) {
-    val = rcdb_reader_read_data(cdb);
+    val = RCDB_READER_READ(data);
   }
 
   return val;
@@ -307,9 +310,7 @@ rcdb_reader_fetch_last(VALUE self, VALUE key) {
   }
 
   if (pos > 0) {
-    val = rb_str_buf_new(len);
-    cdb_read(cdb, RSTRING_PTR(val), len, pos);
-    rb_str_set_len(val, len);
+    RCDB_READER_READ_POS(pos)
   }
 
   return val;
@@ -511,6 +512,8 @@ rcdb_init_reader(void) {
   rb_define_alloc_func(cCDBReader, rcdb_reader_alloc);
   rb_include_module(cCDBReader, rb_mEnumerable);
 
+  rb_define_attr(cCDBReader, "encoding", 1, 1);
+
   rb_define_method(cCDBReader, "close",       rcdb_reader_close,        0);
   rb_define_method(cCDBReader, "closed?",     rcdb_reader_closed_p,     0);
   rb_define_method(cCDBReader, "dump",        rcdb_reader_dump,         0);

data/ext/libcdb/ruby_cdb_reader.h

@@ -28,19 +28,28 @@
 
 #define RCDB_READER_DEFINE_READ(what) \
 static VALUE \
-rcdb_reader_read_##what(struct cdb *cdb) {\
-  size_t len;\
-  VALUE ret;\
+rcdb_reader_read_##what(struct cdb *cdb, VALUE self) {\
+  size_t len = cdb_##what##len(cdb);\
+  VALUE val;\
 \
-  len = cdb_##what##len(cdb);\
-  ret = rb_str_buf_new(len);\
+  RCDB_READER_READ_POS(cdb_##what##pos(cdb))\
 \
-  cdb_read(cdb, RSTRING_PTR(ret), len, cdb_##what##pos(cdb));\
-  rb_str_set_len(ret, len);\
-\
-  return ret;\
+  return val;\
 }
 
+#define RCDB_READER_READ_POS(pos) \
+  val = rb_str_buf_new(len);\
+\
+  cdb_read(cdb, RSTRING_PTR(val), len, pos);\
+  rb_str_set_len(val, len);\
+\
+  rb_funcall(val, rb_intern("force_encoding"), 1, rb_iv_get(self, "@encoding"));
+
+#define RCDB_READER_READ(what) rcdb_reader_read_##what(cdb, self)
+
+#define RCDB_READER_STRING_LEN(str) \
+  rb_funcall(LONG2NUM(RSTRING_LEN(str)), rb_intern("to_s"), 0)
+
 extern VALUE cCDBReader;
 void rcdb_init_reader(void);
 

data/ext/libcdb/ruby_cdb_writer.c

@@ -60,6 +60,8 @@ static void
 rcdb_writer_put_value(struct cdb_make *cdbm, VALUE key, VALUE val, enum cdb_put_mode mode) {
   long i;
 
+  StringValue(key);
+
   switch (TYPE(val)) {
     case T_ARRAY:
       switch (mode) {
@@ -132,8 +134,7 @@ rcdb_writer_put(int argc, VALUE *argv, VALUE self, enum cdb_put_mode mode) {
 
           break;
         case T_HASH:
-          val = rb_ary_new();
-          st_foreach(RHASH_TBL(arg), rcdb_writer_push_pair, val);
+          rb_hash_foreach(arg, rcdb_writer_push_pair, val = rb_ary_new());
 
           for (i = 0; i < RARRAY_LEN(val); i++) {
             tmp = rb_ary_entry(val, i);

data/lib/libcdb/version.rb

@@ -5,8 +5,8 @@ module LibCDB
     module Version
 
       MAJOR = 0
-      MINOR = 1
-      TINY  = 1
+      MINOR = 2
+      TINY  = 0
 
       class << self
 

data/lib/libcdb.rb

@@ -86,6 +86,116 @@ module LibCDB
         }
       end
 
+      # call-seq:
+      #   CDB.load(path, dump) -> aCDB
+      #
+      # Opens +path+ for writing and loads +dump+ into the database. +dump+
+      # may be a string or an IO object. Returns the (unclosed) CDB object.
+      def load(path, dump)
+        require 'strscan'
+
+        s, n, e = nil, 0, lambda { |m| s.eos? ?
+          raise("Unexpected end of input (#{m} at #{n}).") :
+          raise("#{m} at #{n}:#{s.pos}: #{s.peek(16).inspect}") }
+
+        cdb = open(path, 'w+')
+
+        dump.each_line { |line|
+          n += 1
+
+          s = StringScanner.new(line)
+
+          e['Record identifier expected'] unless s.scan(/\+/)
+
+          e['Key length expected'] unless s.scan(/\d+/)
+          klen = s.matched.to_i
+
+          e['Length separator expected'] unless s.scan(/,/)
+
+          e['Value length expected'] unless s.scan(/\d+/)
+          vlen = s.matched.to_i
+
+          e['Key separator expected'] unless s.scan(/:/)
+
+          key = ''
+          klen.times { key << s.get_byte }
+
+          e['Value separator expected'] unless s.scan(/->/)
+
+          value = ''
+          vlen.times { value << s.get_byte }
+
+          e['Record terminator expected'] unless s.scan(/\n/)
+          e['Unexpected data'] unless s.eos?
+
+          cdb.store(key, value)
+        }
+
+        cdb
+      end
+
+      # call-seq:
+      #   CDB.load_file(path, file) -> aCDB
+      #
+      # Loads the dump at +file+ into the database at +path+ (see #load).
+      def load_file(path, file)
+        File.open(file, 'rb') { |f| self.load(path, f) }
+      end
+
+      # call-seq:
+      #   CDB.stats(path) -> aHash
+      #
+      # Returns a hash with the stats on +path+.
+      def stats(path)
+        {}.tap { |stats| open(path) { |cdb|
+          stats[:records] = cnt = cdb.total
+
+          stats[:keys]   = khash = { min: Float::INFINITY, avg: 0, max: 0 }
+          stats[:values] = vhash = khash.dup
+
+          stats[:hash] = Hash.new(0).update(distances: Hash.new([0, 0]))
+
+          khash[:min] = vhash[:min] = 0 and break if cnt.zero?
+
+          ktot, vtot, update = 0, 0, lambda { |h, s| s.bytesize.tap { |l|
+            h[:min] = l if l < h[:min]
+            h[:max] = l if l > h[:max]
+          } }
+
+          cdb.each_key   { |k| ktot += update[khash, k] }
+          cdb.each_value { |v| vtot += update[vhash, v] }
+
+          khash[:avg] = (ktot + cnt / 2) / cnt
+          vhash[:avg] = (vtot + cnt / 2) / cnt
+
+          # TODO: hash table stats
+        } }
+      end
+
+      # call-seq:
+      #   CDB.print_stats(path) -> aHash
+      #
+      # Prints the #stats on +path+.
+      def print_stats(path)
+        stats(path).tap { |s|
+          r, k, v, h = s.values_at(:records, :keys, :values, :hash)
+
+          v1, v2 = [:min, :avg, :max], [:tables, :entries, :collisions]
+
+          puts 'number of records: %d'                    % r
+          puts 'key min/avg/max length: %d/%d/%d'         % k.values_at(*v1)
+          puts 'val min/avg/max length: %d/%d/%d'         % v.values_at(*v1)
+          next # TODO: hash table stats
+          puts 'hash tables/entries/collisions: %d/%d/%d' % h.values_at(*v2)
+          puts 'hash table min/avg/max length: %d/%d/%d'  % h.values_at(*v1)
+          puts 'hash table distances:'
+
+          d = h[:distances]
+          0.upto(9) { |i| puts ' d%d: %6d %2d%%' % [i, *d[i]] }
+          puts ' >9: %6d %2d%%' % d[-1]
+        }
+      end
+
       private
 
       def _open_args(path, mode)
@@ -145,11 +255,11 @@ module LibCDB
     end
 
     def_delegators :reader, :[], :dump, :each, :each_dump, :each_key,
-                            :each_value, :empty?, :fetch, :fetch_all,
-                            :fetch_first, :fetch_last, :get, :has_key?,
-                            :has_value?, :include?, :key, :key?, :keys,
-                            :length, :member?, :rget, :size, :to_a,
-                            :to_h, :total, :value?, :values, :values_at
+                            :each_value, :empty?, :encoding, :encoding=,
+                            :fetch, :fetch_all, :fetch_first, :fetch_last,
+                            :get, :has_key?, :has_value?, :include?, :key,
+                            :key?, :keys, :length, :member?, :rget, :size,
+                            :to_a, :to_h, :total, :value?, :values, :values_at
 
     def_delegators :writer, :<<, :[]=, :add, :insert, :replace, :store