szaru 0.1.1

data/ext/Makefile

@@ -0,0 +1,157 @@
+
+SHELL = /bin/sh
+
+#### Start of system configuration section. ####
+
+srcdir = .
+topdir = /usr/local/lib/ruby/1.8/i686-linux
+hdrdir = $(topdir)
+VPATH = $(srcdir):$(topdir):$(hdrdir)
+exec_prefix = $(prefix)
+prefix = $(DESTDIR)/usr/local
+sharedstatedir = $(prefix)/com
+mandir = $(datarootdir)/man
+psdir = $(docdir)
+oldincludedir = $(DESTDIR)/usr/include
+localedir = $(datarootdir)/locale
+bindir = $(exec_prefix)/bin
+libexecdir = $(exec_prefix)/libexec
+sitedir = $(libdir)/ruby/site_ruby
+htmldir = $(docdir)
+vendorarchdir = $(vendorlibdir)/$(sitearch)
+includedir = $(prefix)/include
+infodir = $(datarootdir)/info
+vendorlibdir = $(vendordir)/$(ruby_version)
+sysconfdir = $(prefix)/etc
+libdir = $(exec_prefix)/lib
+sbindir = $(exec_prefix)/sbin
+rubylibdir = $(libdir)/ruby/$(ruby_version)
+docdir = $(datarootdir)/doc/$(PACKAGE)
+dvidir = $(docdir)
+vendordir = $(libdir)/ruby/vendor_ruby
+datarootdir = $(prefix)/share
+pdfdir = $(docdir)
+archdir = $(rubylibdir)/$(arch)
+sitearchdir = $(sitelibdir)/$(sitearch)
+datadir = $(datarootdir)
+localstatedir = $(prefix)/var
+sitelibdir = $(sitedir)/$(ruby_version)
+
+CC = gcc
+LIBRUBY = $(LIBRUBY_A)
+LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
+LIBRUBYARG_SHARED = -Wl,-R -Wl,$(libdir) -L$(libdir) 
+LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static
+
+RUBY_EXTCONF_H = 
+CFLAGS   =  -fPIC -g -O2 $(cflags) 
+INCFLAGS = -I. -I. -I/usr/local/lib/ruby/1.8/i686-linux -I.
+DEFS     = -D_FILE_OFFSET_BITS=64
+CPPFLAGS =   -D_FILE_OFFSET_BITS=64 
+CXXFLAGS = $(CFLAGS) 
+ldflags  = -L.  -rdynamic -Wl,-export-dynamic
+dldflags = 
+archflag = 
+DLDFLAGS = $(ldflags) $(dldflags) $(archflag)
+LDSHARED = $(CC) -shared
+AR = ar
+EXEEXT = 
+
+RUBY_INSTALL_NAME = ruby
+RUBY_SO_NAME = ruby
+arch = i686-linux
+sitearch = i686-linux
+ruby_version = 1.8
+ruby = /usr/local/bin/ruby
+RUBY = $(ruby)
+RM = rm -f
+MAKEDIRS = mkdir -p
+INSTALL = /usr/bin/install -c
+INSTALL_PROG = $(INSTALL) -m 0755
+INSTALL_DATA = $(INSTALL) -m 644
+COPY = cp
+
+#### End of system configuration section. ####
+
+preload = 
+
+libpath = . $(libdir)
+LIBPATH =  -L. -L$(libdir) -Wl,-R$(libdir)
+DEFFILE = 
+
+CLEANFILES = mkmf.log
+DISTCLEANFILES = 
+
+extout = 
+extout_prefix = 
+target_prefix = 
+LOCAL_LIBS = 
+LIBS =  -lszaru  -lrt -ldl -lcrypt -lm   -lc
+SRCS = rb_szaru.cc
+OBJS = rb_szaru.o
+TARGET = szaru
+DLLIB = $(TARGET).so
+EXTSTATIC = 
+STATIC_LIB = 
+
+BINDIR        = $(bindir)
+RUBYCOMMONDIR = $(sitedir)$(target_prefix)
+RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
+RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
+
+TARGET_SO     = $(DLLIB)
+CLEANLIBS     = $(TARGET).so $(TARGET).il? $(TARGET).tds $(TARGET).map
+CLEANOBJS     = *.o *.a *.s[ol] *.pdb *.exp *.bak
+
+all:		$(DLLIB)
+static:		$(STATIC_LIB)
+
+clean:
+		@-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES)
+
+distclean:	clean
+		@-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
+		@-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
+
+realclean:	distclean
+install: install-so install-rb
+
+install-so: $(RUBYARCHDIR)
+install-so: $(RUBYARCHDIR)/$(DLLIB)
+$(RUBYARCHDIR)/$(DLLIB): $(DLLIB)
+	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
+install-rb: pre-install-rb install-rb-default
+install-rb-default: pre-install-rb-default
+pre-install-rb: Makefile
+pre-install-rb-default: Makefile
+$(RUBYARCHDIR):
+	$(MAKEDIRS) $@
+
+site-install: site-install-so site-install-rb
+site-install-so: install-so
+site-install-rb: install-rb
+
+.SUFFIXES: .c .m .cc .cxx .cpp .C .o
+
+.cc.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.cxx.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.cpp.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.C.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.c.o:
+	$(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $<
+
+$(DLLIB): $(OBJS) Makefile
+	@-$(RM) $@
+	$(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
+
+
+
+$(OBJS): ruby.h defines.h

data/ext/converter.h

@@ -0,0 +1,69 @@
+// Copyright 2010 Yuji Kaneda
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//      http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ------------------------------------------------------------------------
+
+// specialization of Conveter only for int32_t, int64_t, double
+template< typename Value >
+class Converter {
+public:
+  static inline VALUE ToRuby(Value value);
+  static inline Value FromRuby(VALUE value);
+  static inline void CheckType(VALUE value);
+};
+
+template<> inline 
+VALUE Converter<int32_t>::ToRuby(int32_t value){ 
+  return INT2FIX(value);
+}
+
+template<> inline 
+VALUE Converter<int64_t>::ToRuby(int64_t value){ 
+  return LONG2FIX(value);
+}
+
+template<> inline 
+VALUE Converter<double>::ToRuby(double value){ 
+  return rb_float_new(value);
+}
+
+template<> inline 
+int32_t Converter<int32_t>::FromRuby(VALUE value){ 
+  return FIX2INT(value);
+}
+
+template<> inline 
+int64_t Converter<int64_t>::FromRuby(VALUE value){ 
+  return FIX2LONG(value);
+}
+
+template<> inline 
+double Converter<double>::FromRuby(VALUE value){ 
+  return RFLOAT(value)->value;
+}
+
+template<> inline 
+void Converter<int32_t>::CheckType(VALUE value){ 
+  Check_Type(value, T_FIXNUM);
+}
+
+template<> inline 
+void Converter<int64_t>::CheckType(VALUE value){ 
+  Check_Type(value, T_FIXNUM);
+}
+
+template<> inline 
+void Converter<double>::CheckType(VALUE value){ 
+  Check_Type(value, T_FLOAT);
+}
+

data/ext/extconf.rb

@@ -0,0 +1,5 @@
+require "mkmf"
+dir_config('szaru')
+if have_library('szaru')
+  create_makefile('szaru')
+end

data/ext/rb_szaru.cc

@@ -0,0 +1,268 @@
+// Copyright 2010 Yuji Kaneda
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//      http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ------------------------------------------------------------------------
+
+// Ruby Binding of SZaru
+
+#include <ruby.h>
+#include <szaru.h>
+// local include file
+#include "converter.h"
+
+extern "C" {
+  void Init_szaru(void);
+}
+
+namespace {
+
+template<typename Value>
+class RubyQuantileEstimator {
+private:
+  static void
+  Free(SZaru::QuantileEstimator<Value> **ptr)
+  {
+    if (*ptr) {
+      delete *ptr;
+    }
+  }
+  
+  static VALUE
+  Alloc(VALUE klass)
+  {
+    SZaru::QuantileEstimator<Value> **ptr = ALLOC(SZaru::QuantileEstimator<Value>*);
+    *ptr = NULL;
+    return Data_Wrap_Struct(klass, 0, Free, ptr);
+  }
+  
+  static VALUE
+  Initialize(VALUE self, VALUE maxElems)
+  {
+    SZaru::QuantileEstimator<Value> **ptr;
+    Data_Get_Struct(self, SZaru::QuantileEstimator<Value>*, ptr);
+    *ptr = SZaru::QuantileEstimator<Value>::Create(NUM2LONG(maxElems));
+    return Qnil;
+  }
+  
+  static VALUE
+  AddElem(VALUE self, VALUE elem)
+  {
+    SZaru::QuantileEstimator<Value> **qe;
+    Converter<Value>::CheckType(elem);
+    Data_Get_Struct(self, SZaru::QuantileEstimator<Value>*, qe);
+    (*qe)->AddElem(Converter<Value>::FromRuby(elem));
+    return Qnil;
+  }
+  
+  static VALUE
+  Estimate(VALUE self)
+  {
+    SZaru::QuantileEstimator<Value> **qe;
+    Data_Get_Struct(self, SZaru::QuantileEstimator<Value>*, qe);
+    std::vector<Value> quantiles;
+    (*qe)->Estimate(quantiles);
+    VALUE ary = rb_ary_new2(quantiles.size());
+    for (int i = 0; i < quantiles.size(); i++) {
+      rb_ary_push(ary, Converter<Value>::ToRuby(quantiles[i]));
+    }
+    return ary;
+  }
+
+public:
+
+  static VALUE 
+  Define(VALUE superModule, const char *name)
+  {
+    VALUE cQuantileEstimator = rb_define_class_under(superModule, name, rb_cObject);
+    rb_define_alloc_func(cQuantileEstimator, Alloc);
+    rb_define_private_method(cQuantileEstimator, "initialize", 
+			     RUBY_METHOD_FUNC(Initialize), 1);
+    rb_define_method(cQuantileEstimator, "add_elem", 
+		     RUBY_METHOD_FUNC(AddElem), 1);
+    rb_define_method(cQuantileEstimator, "estimate", 
+		     RUBY_METHOD_FUNC(Estimate), 0);
+    return cQuantileEstimator;
+  }
+};
+
+
+template< typename Value >
+class RubyTopEstimator {
+private:
+
+  static void
+  Free(SZaru::TopEstimator<Value> **ptr)
+  {
+    if (*ptr) {
+      delete *ptr;
+    }
+  }
+  
+  static VALUE
+  Alloc(VALUE klass)
+  {
+    SZaru::TopEstimator<Value> **ptr = ALLOC(SZaru::TopEstimator<Value>*);
+    *ptr = NULL;
+    return Data_Wrap_Struct(klass, 0, Free, ptr);
+  }
+
+  static VALUE
+  Initialize(VALUE self, VALUE maxElems)
+  {
+    SZaru::TopEstimator<Value> **ptr;
+    Check_Type(maxElems, T_FIXNUM);
+    Data_Get_Struct(self, SZaru::TopEstimator<Value>*, ptr);
+    *ptr = SZaru::TopEstimator<Value>::Create(NUM2LONG(maxElems));
+    return Qnil;
+  }
+  
+  static VALUE
+  AddElem(VALUE self, VALUE elem)
+  {
+    SZaru::TopEstimator<Value> **te;
+    Check_Type(elem, T_STRING);
+    Data_Get_Struct(self, SZaru::TopEstimator<Value>*, te);
+    (*te)->AddElem(std::string(RSTRING_PTR(elem), RSTRING_LEN(elem)));
+    return Qnil;
+  }
+
+  static VALUE
+  AddWeightedElem(VALUE self, VALUE elem, VALUE weight)
+  {
+    SZaru::TopEstimator<Value> **te;
+    Check_Type(elem, T_STRING);
+    Converter<Value>::CheckType(weight);
+    Data_Get_Struct(self, SZaru::TopEstimator<Value>*, te);
+    (*te)->AddWeightedElem(std::string(RSTRING_PTR(elem), RSTRING_LEN(elem)), 
+			   Converter<Value>::FromRuby(weight));
+    return Qnil;
+  }
+  
+  static VALUE
+  Estimate(VALUE self)
+  {
+    SZaru::TopEstimator<Value> **te;
+    Data_Get_Struct(self, SZaru::TopEstimator<Value>*, te);
+    std::vector<typename SZaru::TopEstimator<Value>::Elem> topElems;
+    (*te)->Estimate(topElems);
+    VALUE ary = rb_ary_new2(topElems.size());
+    for (int i = 0; i < topElems.size(); i++) {
+      rb_ary_push(ary, rb_ary_new3(2, 
+				   rb_str_new(topElems[i].value.c_str(), topElems[i].value.size()), 
+				   Converter<Value>::ToRuby(topElems[i].weight)));
+    }
+    return ary;
+  }
+
+public:
+  static VALUE 
+  Define(VALUE superModule, const char *name)
+  {
+    VALUE cTopEstimator = rb_define_class_under(superModule, name, rb_cObject);
+    rb_define_alloc_func(cTopEstimator, Alloc);
+    rb_define_private_method(cTopEstimator, "initialize", 
+			     RUBY_METHOD_FUNC(Initialize), 1);
+    rb_define_method(cTopEstimator, "add_elem", 
+		     RUBY_METHOD_FUNC(AddElem), 1);
+    rb_define_method(cTopEstimator, "add_weighted_elem", 
+		     RUBY_METHOD_FUNC(AddWeightedElem), 2);
+    rb_define_method(cTopEstimator, "estimate", 
+		     RUBY_METHOD_FUNC(Estimate), 0);
+    return cTopEstimator;
+  }
+
+};
+
+
+class RubyUniqueEstimator {
+private:
+  static void
+  Free(SZaru::UniqueEstimator **ptr)
+  {
+    if (*ptr) {
+      delete *ptr;
+    }
+  }
+  
+  static VALUE
+  Alloc(VALUE klass)
+  {
+    SZaru::UniqueEstimator **ptr = ALLOC(SZaru::UniqueEstimator*);
+    *ptr = NULL;
+    return Data_Wrap_Struct(klass, 0, Free, ptr);
+  }
+  
+  static VALUE
+  Initialize(VALUE self, VALUE maxElems)
+  {
+    SZaru::UniqueEstimator **ptr;
+    Data_Get_Struct(self, SZaru::UniqueEstimator*, ptr);
+    *ptr = SZaru::UniqueEstimator::Create(NUM2LONG(maxElems));
+    return Qnil;
+  }
+  
+  static VALUE
+  AddElem(VALUE self, VALUE elem)
+  {
+    SZaru::UniqueEstimator **ue;
+    Check_Type(elem, T_STRING);
+    Data_Get_Struct(self, SZaru::UniqueEstimator*, ue);
+    (*ue)->AddElemInCIF(RSTRING_PTR(elem), RSTRING_LEN(elem));
+    return Qnil;
+  }
+  
+  static VALUE
+  Estimate(VALUE self)
+  {
+    SZaru::UniqueEstimator **ue;
+    Data_Get_Struct(self, SZaru::UniqueEstimator*, ue);
+    uint64_t unique = (*ue)->Estimate();
+    return LONG2NUM(unique);
+  }
+
+public:
+  static VALUE
+  Define(VALUE superModule, const char *name) {
+    VALUE cUniqueEstimator = rb_define_class_under(superModule, name, rb_cObject);
+    rb_define_alloc_func(cUniqueEstimator, Alloc);
+    rb_define_private_method(cUniqueEstimator, "initialize", 
+			     RUBY_METHOD_FUNC(Initialize), 1);
+    rb_define_method(cUniqueEstimator, "add_elem", 
+		     RUBY_METHOD_FUNC(AddElem), 1);
+    rb_define_method(cUniqueEstimator, "estimate", 
+		     RUBY_METHOD_FUNC(Estimate), 0);  
+    return cUniqueEstimator;
+  }
+};
+
+}
+
+
+void
+Init_szaru(void){
+  VALUE mSZaru = rb_define_module("SZaru");
+  RubyUniqueEstimator::Define(mSZaru, "UniqueEstimator");
+
+  // TopEstimator
+  VALUE mTopEstimator = rb_define_module_under(mSZaru, "TopEstimator");
+  RubyTopEstimator<double>::Define(mTopEstimator, "Double");
+  RubyTopEstimator<int32_t>::Define(mTopEstimator, "Int32");
+  RubyTopEstimator<int64_t>::Define(mTopEstimator, "Int64");
+  
+  // QuantileEstimator
+  VALUE mQuantileEstimator = rb_define_module_under(mSZaru, "QuantileEstimator");
+  RubyQuantileEstimator<double>::Define(mQuantileEstimator, "Double");
+  RubyQuantileEstimator<int32_t>::Define(mQuantileEstimator, "Int32");
+  RubyQuantileEstimator<int64_t>::Define(mQuantileEstimator, "Int64");
+  
+}

data/overview.rd

@@ -0,0 +1,33 @@
+= Ruby Binding of SZaru
+
+== Introduction
+{SZaru}[http://llamerada.github.com/SZaru/] is a library to use {Sawzall}[http://code.google.com/p/szl/] aggregators in pure C++, Ruby and Python.
+Currently, I have implemented the following 3 aggregators:
+[Top] Statistical samplings that record the 'top N' data items based on CountSketch algorithm from "Finding Frequent Items in Data Streams", Moses Charikar, Kevin Chen and Martin Farach-Colton, 2002.
+[Unique] Statistical estimators for the total number of unique data items.
+[Quantile] Approximate N-tiles for data items from an ordered domain based on the following paper: Munro & Paterson, "Selection and Sorting with Limited Storage", Theoretical Computer Science, Vol 12, p 315-323, 1980.
+
+== Example
+ require "szaru"
+ unq_est = SZaru::UniqueEstimator.new(10)
+ 1000.times do |i|
+   unq_est.add_elem(i.to_s + "test")
+ end
+ puts unq_est.estimate # => 913
+
+== License
+
+ Copyright 2010 Yuji Kaneda
+ 
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ 
+      http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ 

data/sample/sample.rb

@@ -0,0 +1,68 @@
+begin
+  require "rubygems"
+  require "szaru"
+rescue
+  puts "load local library"
+  require File.join(File.dirname(__FILE__), "../ext/szaru")
+end
+
+def test_unique
+  ue = SZaru::UniqueEstimator.new(10)
+  1000.times do |i|
+    ue.add_elem(i.to_s + "test")
+  end
+  puts ue.estimate
+end
+
+def test_top
+  te = SZaru::TopEstimator::Int32.new(10)
+  ary = []
+  100.times do |i|
+    i.times do |j|
+      ary << "test#{i}"
+    end
+  end
+  ary.sort_by{rand}.each do |e|
+    te.add_elem(e)
+  end
+  p te.estimate
+end
+
+def test_top2
+  te = SZaru::TopEstimator::Double.new(10)
+  ary = []
+  100.times do |i|
+    ary << ["test#{i}", i]
+  end
+  ary.sort_by{rand}.each do |e, w|
+    te.add_weighted_elem(e, w.to_f)
+  end
+  p te.estimate
+end
+
+def test_quantile
+  te = SZaru::QuantileEstimator::Int64.new(10)
+  ary = []
+  1000.times do |i|
+    # i.times do |j|
+    ary << i
+  end
+  rand_ary  = ary.sort_by{rand}
+  # $stdout.sync = true
+  # puts "start"
+  rand_ary.each_with_index do |e, i|
+    te.add_elem(e)
+  end
+  p te.estimate
+end
+
+
+
+# test_topheap
+# test_sketch
+test_unique
+test_top
+test_top2
+test_quantile
+
+

data/szaru-doc.rb

@@ -0,0 +1,189 @@
+#--
+# SZaru: Porting of excellent Sawzall aggregators. 
+#++
+#:include:overview.rd
+
+
+# SZaru namespace
+module SZaru
+  # Statistical estimators for the total number of unique data items.
+  class UniqueEstimator
+    # Create a UniqueEstimator object.
+    #
+    # _max_elems_ is a tuning parameter.
+    # If _max_elems_ is bigger, the estimation becomes more accurate but consuming more memory.
+    def initialize(max_elems)
+      # (native code)
+    end
+
+    # Add a new element to this entry.
+    # _element_ must be String object.
+    def add_elem(elem)
+      # (native code) 
+    end
+
+    # Return the stimation the number of unique entries.
+    def estimate
+      # (native code) 
+    end
+  end # UniqueEstimator
+
+  # Statistical samplings that record the 'top N' data items.
+  module TopEstimator
+    # TopEstimator of that weight is int32
+    class Int32
+      # Create a TopEstimator::Int32 object.
+      #
+      # _top_elems_ is a number of top elements to be estimate.
+      def initialize(top_elems)
+        # (native code)
+      end
+      
+      # Add a new element to this entry.
+      # _element_ must be String object.
+      def add_elem(elem)
+        # (native code)
+      end
+
+      # Add a new weighted element to this entry.
+      # _element_ must be String object. 
+      # _weight_ msut be Fixnum object.
+      def add_weighted_elem(elem, weight)
+        # (native code)
+      end
+
+      # Return a top elements with weight.
+      # Example: [["abc", 7], ["def", 3]]
+      def estimate
+        # (native code)
+      end
+    end # Int32
+
+    # TopEstimator of that weight is int64
+    class Int64
+      # Create a TopEstimator::Int64 object.
+      #
+      # _top_elems_ is a number of top elements to be estimate.
+      def initialize(top_elems)
+        # (native code)
+      end
+      
+      # Add a new element to this entry.
+      # _element_ must be String object.
+      def add_elem(elem)
+        # (native code)
+      end
+
+      # Add a new weighted element to this entry.
+      # _element_ must be String object. 
+      # _weight_ msut be Fixnum object.
+      def add_weighted_elem(elem, weight)
+        # (native code)
+      end
+
+      # Return a top elements with weight.
+      # Example: [["abc", 7], ["def", 3]]
+      def estimate
+        # (native code)
+      end 
+    end # Int64
+
+    # TopEstimator of that weight is Double
+    class Double
+      # Create a TopEstimator::Double object.
+      #
+      # _top_elems_ is a number of top elements to be estimate.
+      def initialize(top_elems)
+        # (native code)
+      end
+      
+      # Add a new element to this entry.
+      # _element_ must be String object.
+      def add_elem(elem)
+        # (native code)
+      end
+
+      # Add a new weighted element to this entry.
+      # _element_ must be String object. 
+      # _weight_ msut be Float object.
+      def add_weighted_elem(elem, weight)
+        # (native code)
+      end
+
+      # Return a top elements with weight.
+      # Example: [["abc", 7.0], ["def", 3.0]]
+      def estimate
+        # (native code)
+      end
+    end # Double
+  end # TopEstimator
+
+  # Approximate N-tiles for data items from an ordered domain.
+  module QuantileEstimator
+    # TopEstimator of that element is int32
+    class Int32
+      # Create a QuantileEstimator::Int32 object.
+      #
+      # _num_quantiles_ is a number of tiles to be estimate.
+      def initialize(num_quantiles)
+        # (native code)
+      end
+
+      # Add a new element to this entry.
+      # _element_ must be Fixnum object.
+      def add_elem(elem)
+        # (native code)
+      end
+
+      # Return a estimated N tiles.
+      # Example: [0, 3, 7, 9]
+      def estimate()
+        # (native code)
+      end
+    end # Int32
+
+    # TopEstimator of that element is int64
+    class Int64
+      # Create a QuantileEstimator::Int64 object.
+      #
+      # _num_quantiles_ is a number of tiles to be estimate.
+      def initialize(num_quantiles)
+        # (native code)
+      end
+
+      # Add a new element to this entry.
+      # _element_ must be Fixnum object.
+      def add_elem(elem)
+        # (native code)
+      end
+
+      # Return a estimated N tiles.
+      # Example: [0, 3, 7, 9]
+      def estimate()
+        # (native code)
+      end
+    end # Int64
+
+    # TopEstimator of that element is double
+    class Double
+      # Create a QuantileEstimator::Double object.
+      #
+      # _num_quantiles_ is a number of tiles to be estimate.
+      def initialize(num_quantiles)
+        # (native code)
+      end
+
+      # Add a new element to this entry.
+      # _element_ must be Fixnum object.
+      def add_elem(elem)
+        # (native code)
+      end
+
+      # Return a estimated N tiles.
+      # Example: [0.0, 3.2, 6.8, 9.5]
+      def estimate()
+        # (native code)
+      end
+    end # Double
+  end # QuantileEstimator
+end # SZaru

data/test/quantile_spec.rb

@@ -0,0 +1,72 @@
+require File.join(File.dirname(__FILE__), "../ext/szaru")
+
+include SZaru
+quantile_estimators = [QuantileEstimator::Int32, QuantileEstimator::Int64, QuantileEstimator::Double]
+
+def convert_value_from_index(index, value)
+  case index
+  when 0, 1
+    value.to_i
+  when 2
+    value.to_f
+  end
+end
+
+quantile_estimators.each_with_index do |quantile_estimator, te_index|
+  describe quantile_estimator do
+    it "return [0] if no addition exists" do
+      te = quantile_estimator.new(10)
+      te.estimate.should == [0]
+    end
+
+    it "return [min, max] if quantile_elems is 0" do
+      te = quantile_estimator.new(0)
+      te.add_elem(convert_value_from_index(te_index, 10))
+      te.add_elem(convert_value_from_index(te_index, 7))
+      te.estimate.should == [7, 10]
+    end
+
+    it "return exact quantile when the number of elements is small than quantile_elems" do
+      te = quantile_estimator.new(10)
+      n_elemnts = 5
+      n_elemnts.times do |i|
+        te.add_elem(convert_value_from_index(te_index, i))
+      end
+      quantile_elements = te.estimate
+      expexcted_values = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
+      expexcted = expexcted_values.map{|value| convert_value_from_index(te_index, value)}
+      quantile_elements.should == expexcted
+    end
+
+    it "return same result when calling estimate twice" do
+      te = quantile_estimator.new(10)
+      n_elemnts = 5
+      n_elemnts.times do |i|
+        te.add_elem(convert_value_from_index(te_index, i))
+      end
+      # first call
+      quantile_elements = te.estimate
+      # second call
+      quantile_elements = te.estimate
+      expexcted_values = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
+      expexcted = expexcted_values.map{|value| convert_value_from_index(te_index, value)}
+      quantile_elements.should == expexcted
+    end
+
+    it "return approximate number when the number of elements is greater than quantile_elems" do
+      qe = quantile_estimator.new(11)
+      elems = (0 .. 1000).to_a.sort_by{rand}
+      elems.each do |elem|
+        qe.add_elem(convert_value_from_index(te_index, elem))
+      end
+      quantiles = qe.estimate
+      quantiles.length.should == 11
+      quantiles.each_with_index do |tile, index|
+        exact = index * 100
+        diff = (tile - exact).abs
+        diff.should < 10
+      end
+    end
+    
+  end
+end

data/test/top_spec.rb

@@ -0,0 +1,110 @@
+require File.join(File.dirname(__FILE__), "../ext/szaru")
+
+include SZaru
+top_estimators = [TopEstimator::Int32, TopEstimator::Int64, TopEstimator::Double]
+
+def convert_value_from_index(index, value)
+  case index
+  when 0, 1
+    value.to_i
+  when 2
+    value.to_f
+  end
+end
+
+top_estimators.each_with_index do |top_estimator, te_index|
+  describe top_estimator do
+    it "return [] if no addition exists" do
+      te = top_estimator.new(10)
+      te.estimate.should == []
+    end
+
+    it "return [] if top_elems is 0" do
+      te = top_estimator.new(0)
+      te.add_elem("test")
+      te.estimate.should == []
+    end
+
+    it "return exact number when the number of elements is small than top_elems" do
+      te = top_estimator.new(10)
+      n_elemnts = 5
+      n_elemnts.times do |i|
+        te.add_elem("test#{i}")
+        te.add_weighted_elem("test#{i}", convert_value_from_index(te_index, i))
+      end
+      top_elements = te.estimate
+      top_elements.length.should == n_elemnts
+      n_elemnts.times do |i|
+        # check element
+        top_elements[i][0].should == "test#{n_elemnts - i - 1}"
+        # check weight
+        exact_weight = convert_value_from_index(te_index, n_elemnts - i)
+        top_elements[i][1].should == exact_weight
+      end
+    end
+
+    it "return same result when calling estimate twice" do
+      te = top_estimator.new(10)
+      n_elemnts = 5
+      n_elemnts.times do |i|
+        te.add_elem("test#{i}")
+        te.add_weighted_elem("test#{i}", convert_value_from_index(te_index, i))
+      end
+      # first call
+      top_elements = te.estimate
+      # second call
+      top_elements = te.estimate
+      top_elements.length.should == n_elemnts
+      n_elemnts.times do |i|
+        # check element
+        top_elements[i][0].should == "test#{n_elemnts - i - 1}"
+        # check weight
+        exact_weight = convert_value_from_index(te_index, n_elemnts - i)
+        top_elements[i][1].should == exact_weight
+      end
+    end
+
+    it "return approximate number when the number of elements is greater than top_elems" do
+      te = top_estimator.new(10)
+      n_large_elemnts = 30
+      n_small_elemnts = 1000
+      # create input stream
+      elems = []
+      # large element x_i occurs x_i^2 times.
+      n_large_elemnts.times do |i|
+        (i * i).times do 
+          elems << i
+        end
+      end
+      # small element y_i occurs less than 5 times.
+      n_small_elemnts.times do |i|
+        rand(5).times do 
+          elems << i
+        end
+      end
+      # run input stream 2 times in random oreder
+      2.times do 
+        elems.sort_by{ rand }.each do |j|
+          te.add_elem("test#{j}")
+        end
+      end
+      # check estimation
+      top_elements = te.estimate
+      top_elements.length.should == 10
+      10.times do |i|
+        exact_index = n_large_elemnts - i - 1
+        # check element
+        top_elements[i][0] =~ /test(\d*)/
+        estimated_index = $1.to_i
+        diff = (exact_index - estimated_index).abs
+        diff.should < 3
+        # check weight
+        exact_weight = convert_value_from_index(te_index, 2 * exact_index * exact_index)
+        diff = top_elements[i][1] - exact_weight
+        error = (diff / exact_weight.to_f).abs
+        error.should < 0.1
+      end
+    end
+    
+  end
+end

data/test/unique_spec.rb

@@ -0,0 +1,39 @@
+require File.join(File.dirname(__FILE__), "../ext/szaru")
+
+include SZaru
+describe UniqueEstimator do
+  it "return 0 if no addition exists" do
+    ue = UniqueEstimator.new(10)
+    ue.estimate.should == 0
+  end
+
+  it "return 0 if max_elems is 0" do
+    ue = UniqueEstimator.new(0)
+    ue.add_elem("test")
+    ue.estimate.should == 0
+  end
+
+  it "return exact number when the number of elements is small than max_elems" do
+    ue = UniqueEstimator.new(10)
+    5.times do |i|
+      ue.add_elem("test#{i}")
+    end
+    ue.estimate.should == 5
+  end
+
+  it "return approximate number when the number of elements is greater than max_elems" do
+    ue = UniqueEstimator.new(10)
+    n_unique = 997
+    elems = Array.new(n_unique){|i| ("test#{i}") }
+    # add elems to ue 2 times in random oreder
+    2.times do 
+      elems.sort_by{ rand }.each do |elm|
+        ue.add_elem(elm)
+      end
+    end
+    diff = ue.estimate - n_unique
+    error_rate = ( diff / n_unique.to_f).abs
+    error_rate.should < 0.1
+  end
+  
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification 
+name: szaru
+version: !ruby/object:Gem::Version 
+  hash: 25
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 1
+  version: 0.1.1
+platform: ruby
+authors: 
+- Yuji Kaneda
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-13 00:00:00 +09:00
+default_executable: 
+dependencies: []
+
+description: |
+  Portings of excellent Sawzall aggregators.
+
+email: llamerada@gmail.com
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- ext/rb_szaru.cc
+- ext/converter.h
+- ext/szaru.so
+- ext/Makefile
+- ext/extconf.rb
+- ext/rb_szaru.o
+- test/quantile_spec.rb
+- test/top_spec.rb
+- test/unique_spec.rb
+- sample/sample.rb
+- overview.rd
+- szaru-doc.rb
+has_rdoc: true
+homepage: http://llamerada.github.com/SZaru/
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- szaru-doc.rb
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Portings of excellent Sawzall aggregators
+test_files: 
+- test/quantile_spec.rb
+- test/top_spec.rb
+- test/unique_spec.rb