fairy 0.6.0 → 0.6.5

data/Makefile

@@ -90,6 +90,7 @@ doc/programming-interface.html: doc/programming-interface.rd
 # gem
 gem:
 	gem build fairy.gemspec
+	mv *.gem pkg
 
 # tar archives
 TGZ_FILES = $(SRCS) $(DEEPCONNECTSRCS)

data/bin/fairy

@@ -120,6 +120,16 @@ module Fairy
     def_prop :MARSHAL_QUEUE_MIN_CHUNK_NO
     def_prop :SIZEDMARSHAL_QUEUE_MAX_CHUNK_NO
 
+    def_prop :XMARSHAL_QUEUE_CHUNK_SIZE
+    def_prop :XMARSHAL_QUEUE_MIN_CHUNK_NO
+    def_prop :XMARSHAL_QUEUE_BUFFERS_CACHE_LIMIT
+    def_prop :XMARSHAL_QUEUE_USE_STRING_BUFFER
+    def_prop :XMARSHAL_QUEUE_LOG_MSTORE
+
+    def_prop :XSIZED_QUEUE_CHUNK_SIZE
+    def_prop :XSIZED_QUEUE_QUEUES_LIMIT
+    def_prop :XSIZED_QUEUE_USE_STRING_BUFFER
+
     def_prop :SORTEDQUEUE_POOL_THRESHOLD
     def_prop :SORTEDQUEUE_THRESHOLD
     def_prop :SORTEDQUEUE_SORTBY
@@ -128,6 +138,7 @@ module Fairy
     def_prop :HERE_POOL_THRESHOLD
     #def_prop :OUTPUT_LOCAL_POOL_THRESHOLD
 
+    def_prop :GROUP_BY
     def_prop :GROUP_BY_NO_SEGMENT
     def_prop :GROUP_BY_HASH_MODULE
     def_prop :GROUP_BY_GROUPING_OPTIMIZE
@@ -136,6 +147,8 @@ module Fairy
     def_prop :GROUP_BY_CMSB_THRESHOLD
     def_prop :GROUP_BY_CMSB_CHUNK_SIZE
 
+    def_prop :XGROUP_BY_BUFFERING_POLICY
+
     def_prop :BARRIER_MEMORY_BUFFERING_POLICY
 
     def_prop :SORT_BUFFERING_POLICY
@@ -297,19 +310,32 @@ module Fairy
 
   DEFAULT_CONF.POSTMAPPING_POLICY = nil
 
-  DEFAULT_CONF.PREQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
-  DEFAULT_CONF.POSTQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
+  DEFAULT_CONF.PREQUEUING_POLICY = {:queuing_class => :XSizedQueue}
+  DEFAULT_CONF.POSTQUEUING_POLICY = {:queuing_class => :XMarshaledQueue}
 
   DEFAULT_CONF.POSTQUEUE_MAX_TRANSFER_SIZE = 100000
-  DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD = 10000
+  DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD = 100000
   DEFAULT_CONF.ONMEMORY_SIZEDQUEUE_SIZE = 10000
   DEFAULT_CONF.FILEBUFFEREDQUEUE_THRESHOLD = 10000/2
 
   DEFAULT_CONF.MARSHAL_QUEUE_CHUNK_SIZE = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
   DEFAULT_CONF.MARSHAL_QUEUE_MIN_CHUNK_NO = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
   DEFAULT_CONF.SIZEDMARSHAL_QUEUE_MAX_CHUNK_NO = 10
+
+  DEFAULT_CONF.XMARSHAL_QUEUE_CHUNK_SIZE = DEFAULT_CONF.MARSHAL_QUEUE_CHUNK_SIZE 
+  DEFAULT_CONF.XMARSHAL_QUEUE_MIN_CHUNK_NO = DEFAULT_CONF.MARSHAL_QUEUE_MIN_CHUNK_NO 
+  DEFAULT_CONF.XMARSHAL_QUEUE_BUFFERS_CACHE_LIMIT = 1000
+  DEFAULT_CONF.XMARSHAL_QUEUE_USE_STRING_BUFFER = true
+  DEFAULT_CONF.XMARSHAL_QUEUE_LOG_MSTORE = false
+
+  DEFAULT_CONF.XSIZED_QUEUE_CHUNK_SIZE = DEFAULT_CONF.XMARSHAL_QUEUE_CHUNK_SIZE
+  DEFAULT_CONF.XSIZED_QUEUE_QUEUES_LIMIT = 
+    DEFAULT_CONF.XMARSHAL_QUEUE_BUFFERS_CACHE_LIMIT
+  DEFAULT_CONF.XSIZED_QUEUE_USE_STRING_BUFFER = 
+    DEFAULT_CONF.XMARSHAL_QUEUE_USE_STRING_BUFFER
   
-  DEFAULT_CONF.SORTEDQUEUE_POOL_THRESHOLD = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
+  DEFAULT_CONF.SORTEDQUEUE_POOL_THRESHOLD = 
+    DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
   DEFAULT_CONF.SORTEDQUEUE_THRESHOLD = 10000/2
   DEFAULT_CONF.SORTEDQUEUE_SORTBY = %{|v| v}
 
@@ -317,6 +343,8 @@ module Fairy
   DEFAULT_CONF.HERE_POOL_THRESHOLD = 32000
   #DEFAULT_CONF.OUTPUT_LOCAL_POOL_THRESHOLD = 32000
 
+  DEFAULT_CONF.GROUP_BY = :XGroupBy
+
   DEFAULT_CONF.GROUP_BY_NO_SEGMENT = 4
   DEFAULT_CONF.GROUP_BY_HASH_MODULE = "fairy/share/hash-md5"
   DEFAULT_CONF.GROUP_BY_GROUPING_OPTIMIZE = false
@@ -325,9 +353,11 @@ module Fairy
   DEFAULT_CONF.GROUP_BY_CMSB_THRESHOLD = 400_000
   DEFAULT_CONF.GROUP_BY_CMSB_CHUNK_SIZE = 1000
 
+  DEFAULT_CONF.XGROUP_BY_BUFFERING_POLICY = {:buffering_class => :XDirectMergeSortBuffer}
+
   DEFAULT_CONF.BARRIER_MEMORY_BUFFERING_POLICY = {:queuing_class => :PoolQueue}
 
-  DEFAULT_CONF.SORT_BUFFERING_POLICY = {:buffering_class => "PGroupBy::DirectMergeSortBuffer"}
+  DEFAULT_CONF.SORT_BUFFERING_POLICY = {:buffering_class => "PGroupBy::XDirectMergeSortBuffer"}
   DEFAULT_CONF.SORT_SAMPLING_MIN = 100
   DEFAULT_CONF.SORT_SAMPLING_MAX = 10000
   DEFAULT_CONF.SORT_SAMPLING_RATIO_1_TO = 100

data/ext/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+
+create_makefile("fairy")

data/ext/fairy.c

@@ -0,0 +1,180 @@
+/**********************************************************************
+
+  fairy.c -
+  Copyright (C) 2007-2011 Rakuten, Inc.
+
+**********************************************************************/
+
+#include <stdarg.h>
+
+#include "ruby.h"
+#include "fairy.h"
+
+extern void Init_simple_hash();
+extern void Init_fixnum_buffer();
+extern void Init_string_buffer();
+extern void Init_xsized_queue();
+extern void Init_xmarshaled_queue();
+extern void Init_p_group_by();
+extern void Init_p_xgroup_by();
+
+VALUE rb_mFairy;
+VALUE rb_cFairyLog;
+VALUE rb_cFairyProcessor = Qnil;
+
+VALUE rb_cFairyImport;
+VALUE rb_cFairyExport;
+
+VALUE rb_cFairyImportCTLTOKEN_SET_NO_IMPORT;
+VALUE rb_cFairyImportCTLTOKEN_NULLVALUE;
+VALUE rb_cFairyImportCTLTOKEN_DELAYED_ELEMENT;
+
+VALUE rb_FairyConf;
+VALUE rb_FairyEOS;
+
+static ID id_aref;
+static ID id_def_export;
+
+VALUE
+rb_fairy_conf(const char *conf_attr, VALUE policy, const char *policy_name)
+{
+  VALUE val = Qnil;
+
+  if (policy) {
+    val = rb_funcall(policy, id_aref, 1, ID2SYM(rb_intern(policy_name)));
+  }
+  
+  if (NIL_P(val) && conf_attr != NULL) {
+    val = rb_funcall(rb_FairyConf, rb_intern(conf_attr), 0);
+  }
+  return val;
+}
+
+VALUE
+rb_fairy_processor_def_export(VALUE klass)
+{
+  return rb_funcall(rb_cFairyProcessor, id_def_export, 1, klass);
+}
+
+VALUE
+rb_fairy_processor_def_export2(VALUE klass, char *name)
+{
+  VALUE str = rb_str_new2(name);
+  
+  return rb_funcall(rb_cFairyProcessor, id_def_export, 2, klass, str);
+}
+
+#define DEF_LOG_FUNC(LEVEL) \
+  static ID id_##LEVEL;	    \
+  VALUE						    \
+  rb_fairy_##LEVEL(VALUE sender, const char *message)	\
+  {							\
+    return rb_funcall(rb_cFairyLog, id_##LEVEL, 2,	\
+		      sender, rb_str_new_cstr(message));	\
+  }								\
+  VALUE								\
+  rb_fairy_##LEVEL##f(VALUE sender, const char *format, ...)	\
+  {									\
+    VALUE result;						\
+    va_list ap;								\
+    va_start(ap, format);						\
+    result = rb_vsprintf(format, ap);					\
+    va_end(ap);								\
+    return rb_funcall(rb_cFairyLog, id_##LEVEL, 2, sender, result);	\
+  }									\
+  static ID id_##LEVEL##_exception;					\
+  VALUE									\
+  rb_fairy_##LEVEL##_exception(VALUE sender)				\
+  {									\
+    return rb_funcall(rb_cFairyLog, id_##LEVEL##_exception, 1,		\
+		      sender);						\
+  }									\
+  static ID id_##LEVEL##_backtrace;					\
+  VALUE									\
+  rb_fairy_##LEVEL##_backtrace(VALUE sender)				\
+  {									\
+    return rb_funcall(rb_cFairyLog, id_##LEVEL##_backtrace, 1, sender);	\
+  }
+
+DEF_LOG_FUNC(fatal);
+DEF_LOG_FUNC(error);
+DEF_LOG_FUNC(warn);
+DEF_LOG_FUNC(info);
+DEF_LOG_FUNC(verbose);
+DEF_LOG_FUNC(debug);
+
+#define LOG_PRE ""
+#define LOG_POST "_exception"
+
+#define DEF_LOG_ID(level) \
+  id_##level = rb_intern(#level); \
+  id_##level##_exception = rb_intern(LOG_PRE #level LOG_POST); \
+  id_##level##_backtrace = rb_intern("" #level "_backtrace");
+
+static ID id_debug_p;
+
+VALUE
+rb_fairy_debug_p(VALUE obj)
+{
+  return rb_funcall(rb_cFairyLog, id_debug_p, 1, obj);
+}
+
+VALUE
+rb_fairy_debug_p2(VALUE self, const char *pre, VALUE obj)
+{
+  const char post[] = ": %s";
+  char *buf = ALLOCA_N(char, strlen(pre) + strlen(post) + 1);
+  
+  strcpy(buf, pre);
+  strcat(buf, post);
+    
+  return rb_fairy_debugf(self, buf, RSTRING_PTR(rb_inspect(obj)));
+}
+
+Init_fairy()
+{
+  rb_mFairy = rb_define_module("Fairy");
+  rb_FairyConf = rb_const_get(rb_mFairy, rb_intern("CONF"));
+
+  if (rb_const_defined(rb_mFairy, rb_intern("Processor"))) {
+      rb_cFairyProcessor = rb_const_get(rb_mFairy, rb_intern("Processor"));
+  }
+
+  rb_require("fairy/share/port");
+
+  rb_cFairyLog = rb_const_get(rb_mFairy, rb_intern("Log"));
+
+  rb_cFairyImport = rb_const_get(rb_mFairy, rb_intern("Import"));
+  rb_cFairyExport = rb_const_get(rb_mFairy, rb_intern("Export"));
+  
+  rb_cFairyImportCTLTOKEN_SET_NO_IMPORT =
+    rb_const_get(rb_cFairyImport, rb_intern("CTLTOKEN_SET_NO_IMPORT"));
+  rb_cFairyImportCTLTOKEN_NULLVALUE =
+    rb_const_get(rb_cFairyImport, rb_intern("CTLTOKEN_NULLVALUE"));
+  rb_cFairyImportCTLTOKEN_DELAYED_ELEMENT =
+    rb_const_get(rb_cFairyImport, rb_intern("CTLTOKEN_DELAYED_ELEMENT"));
+		 
+  rb_FairyEOS = ID2SYM(rb_intern("END_OF_STREAM"));
+
+  id_aref = rb_intern("[]");
+  id_def_export = rb_intern("def_export");
+  
+  DEF_LOG_ID(fatal);
+  DEF_LOG_ID(error);
+  DEF_LOG_ID(warn);
+  DEF_LOG_ID(info);
+  DEF_LOG_ID(verbose);
+  DEF_LOG_ID(debug);
+  
+  id_debug_p = rb_intern("debug_p");
+
+  Init_simple_hash();
+  Init_fixnum_buffer();
+  Init_string_buffer();
+  Init_xsized_queue();
+  Init_xmarshaled_queue();
+  Init_p_group_by();
+  Init_p_xgroup_by();
+
+  rb_fairy_warn(rb_mFairy, "fairy.so initialize OK");
+}

data/ext/fairy.h

@@ -0,0 +1,94 @@
+/**********************************************************************
+
+  fairy.h -
+  Copyright (C) 2007-2011 Rakuten, Inc.
+
+**********************************************************************/
+
+RUBY_EXTERN VALUE rb_mFairy;
+
+RUBY_EXTERN VALUE rb_cFairyImport;
+RUBY_EXTERN VALUE rb_cFairyExport;
+
+RUBY_EXTERN VALUE rb_mFairySimpleHash;
+RUBY_EXTERN VALUE rb_cFairyFixnumBuffer;
+RUBY_EXTERN VALUE rb_cFairyStringBuffer;
+RUBY_EXTERN VALUE rb_cFairyXMarshaledQueue;
+RUBY_EXTERN VALUE rb_cFairyPXGroupBy;
+
+RUBY_EXTERN VALUE rb_cFairyImportCTLTOKEN_SET_NO_IMPORT;
+RUBY_EXTERN VALUE rb_cFairyImportCTLTOKEN_NULLVALUE;
+RUBY_EXTERN VALUE rb_cFairyImportCTLTOKEN_DELAYED_ELEMENT;
+
+RUBY_EXTERN VALUE rb_FairyConf;
+
+RUBY_EXTERN VALUE rb_fairy_conf(const char *, VALUE, const char *);
+
+RUBY_EXTERN VALUE rb_fairy_processor_def_export(VALUE);
+RUBY_EXTERN VALUE rb_fairy_processor_def_export2(VALUE, char*);
+
+RUBY_EXTERN VALUE rb_fairy_simple_hash(VALUE, VALUE);
+RUBY_EXTERN unsigned int rb_fairy_simple_hash_uint(VALUE, VALUE);
+
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_new(void);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_clear(VALUE);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_length(VALUE);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_realsize(VALUE);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_push_long(VALUE, long);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_push(VALUE, VALUE);
+RUBY_EXTERN long rb_fairy_fixnum_buffer_pop_long(VALUE);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_pop(VALUE);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_each(VALUE);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_each_callback(VALUE, VALUE(*)(long, VALUE), VALUE);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_to_a(VALUE);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_marshal_dump(VALUE);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_marshal_load(VALUE, VALUE);
+RUBY_EXTERN VALUE rb_fairy_fixnum_buffer_inspect(VALUE);
+
+RUBY_EXTERN VALUE rb_fairy_string_buffer_new(void);
+RUBY_EXTERN VALUE rb_fairy_string_buffer_clear(VALUE);
+RUBY_EXTERN VALUE rb_fairy_string_buffer_size(VALUE);
+RUBY_EXTERN VALUE rb_fairy_string_buffer_push(VALUE, VALUE);
+RUBY_EXTERN VALUE rb_fairy_string_buffer_to_a(VALUE);
+RUBY_EXTERN VALUE rb_fairy_string_buffer_marshal_dump(VALUE);
+RUBY_EXTERN VALUE rb_fairy_string_buffer_marshal_load(VALUE, VALUE);
+
+RUBY_EXTERN VALUE rb_FairyEOS;
+
+#define EOS_P(v) (v == rb_FairyEOS)
+
+RUBY_EXTERN VALUE rb_fairy_xmarshaled_queue_new(VALUE, VALUE, VALUE);
+RUBY_EXTERN VALUE rb_fairy_xmarshaled_queue_push(VALUE, VALUE);
+RUBY_EXTERN VALUE rb_fairy_xmarshaled_queue_pop(VALUE);
+RUBY_EXTERN VALUE rb_fairy_xmarshaled_queue_inspect(VALUE);
+
+
+#define DEF_LOG_FUNC_EXTERN(LEVEL)			   \
+  RUBY_EXTERN VALUE rb_fairy_##LEVEL(VALUE, const char *);		\
+  RUBY_EXTERN VALUE rb_fairy_##LEVEL##f(VALUE, const char *, ...);	\
+  RUBY_EXTERN VALUE rb_fairy_##LEVEL##_exception(VALUE);		\
+  RUBY_EXTERN VALUE rb_fairy_##LEVEL##_backtrace(VALUE);
+
+DEF_LOG_FUNC_EXTERN(fatal);
+DEF_LOG_FUNC_EXTERN(error);
+DEF_LOG_FUNC_EXTERN(warn);
+DEF_LOG_FUNC_EXTERN(info);
+DEF_LOG_FUNC_EXTERN(verbose);
+DEF_LOG_FUNC_EXTERN(debug);
+
+
+RUBY_EXTERN VALUE rb_fairy_debug_p(VALUE);
+RUBY_EXTERN VALUE rb_fairy_debug_p2(VALUE, const char *, VALUE);
+
+
+
+
+
+
+
+
+
+
+
+
+

data/ext/fiber_mon.h

@@ -0,0 +1,32 @@
+/**********************************************************************
+
+  fiber-mon.h -
+   	Copyright (C) 2010-2011 Keiju ISHITSUKA
+				(Penta Advanced Labrabries, Co.,Ltd)
+
+**********************************************************************/
+
+RUBY_EXTERN VALUE rb_cFiberMon;
+RUBY_EXTERN VALUE rb_cFiberMonMonitor;
+RUBY_EXTERN VALUE rb_cFiberMonConditionVariable;
+
+RUBY_EXTERN VALUE rb_fibermon_new(void);
+RUBY_EXTERN VALUE rb_fibermon_current(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_entry_fiber(VALUE, VALUE);
+RUBY_EXTERN VALUE rb_fibermon_yield(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_new_mon(VALUE);
+
+RUBY_EXTERN VALUE rb_fibermon_monitor_new(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_monitor_valid_owner_p(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_monitor_try_enter(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_monitor_enter(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_monitor_exit(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_monitor_synchronize(VALUE, VALUE (*)(VALUE), VALUE);
+RUBY_EXTERN VALUE rb_fibermon_monitor_new_cond(VALUE);
+
+RUBY_EXTERN VALUE rb_fibermon_cond_new(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_cond_signal(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_cond_broadcast(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_cond_wait(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_cond_wait_until(VALUE);
+RUBY_EXTERN VALUE rb_fibermon_cond_wait_while(VALUE);

data/ext/fixnum-buffer.c

@@ -0,0 +1,483 @@
+/**********************************************************************
+
+  fixnum-buffer.c -
+  Copyright (C) 2007-2011 Rakuten, Inc.
+
+**********************************************************************/
+
+#include "ruby.h"
+
+#include "fairy.h"
+
+#define FIXNUM_BUFFER_DEFAULT_CAPA 16
+
+VALUE rb_cFairyFixnumBuffer;
+
+typedef struct rb_fairy_fixnum_buffer_struct
+{
+  long capa;
+  long length;
+  long push;
+  long pop;
+  unsigned char *fixnums;
+} fairy_fixnum_buffer_t;
+
+#define GetFairyFixnumBufferPtr(obj, tobj) \
+  TypedData_Get_Struct((obj), fairy_fixnum_buffer_t, &fairy_fixnum_buffer_data_type, (tobj))
+
+static void
+fairy_fixnum_buffer_free(void *ptr)
+{
+  fairy_fixnum_buffer_t *fb = (fairy_fixnum_buffer_t*)ptr;
+
+  if (fb->fixnums) {
+    ruby_xfree(fb->fixnums);
+  }
+  ruby_xfree(ptr);
+}
+
+static size_t
+fairy_fixnum_buffer_memsize(const void *ptr)
+{
+  fairy_fixnum_buffer_t *fb = (fairy_fixnum_buffer_t*)ptr;
+  return ptr ? sizeof(fairy_fixnum_buffer_t) + fb->capa : 0;
+}
+
+#ifdef HAVE_RB_DATA_TYPE_T_FUNCTION
+static const rb_data_type_t fairy_fixnum_buffer_data_type = {
+  "fairy_fixnum_buffer",
+  {0, fairy_fixnum_buffer_free, fairy_fixnum_buffer_memsize,},
+};
+#else
+static const rb_data_type_t fairy_fixnum_buffer_data_type = {
+  "fairy_fixnum_buffer",
+  0,
+  fairy_fixnum_buffer_free,
+  fairy_fixnum_buffer_memsize,
+};
+#endif
+
+static VALUE
+fairy_fixnum_buffer_alloc(VALUE klass)
+{
+  VALUE volatile obj;
+  fairy_fixnum_buffer_t *fb;
+
+  obj = TypedData_Make_Struct(klass, fairy_fixnum_buffer_t, &fairy_fixnum_buffer_data_type, fb);
+  
+  fb->capa = 0;
+  fb->length = 0;
+  fb->push = 0;
+  fb->pop = 0;
+  fb->fixnums = NULL;
+
+  return obj;
+}
+
+static void
+fairy_fixnum_buffer_resize_double_capa(fairy_fixnum_buffer_t *fb)
+{
+  long new_capa = fb->capa * 2;
+
+  REALLOC_N(fb->fixnums, unsigned char, new_capa);
+  
+  if (fb->push > fb->capa) {
+    if (fb->capa - fb->pop <= fb->push - fb->capa) {
+      MEMCPY(&fb->fixnums[fb->pop + fb->capa],
+	     &fb->fixnums[fb->pop], unsigned char, fb->capa - fb->pop);
+      fb->pop += fb->capa;
+      fb->push += fb->capa;
+    }
+    else {
+      MEMCPY(&fb->fixnums[fb->capa],
+	     fb->fixnums, unsigned char, fb->push - fb->capa);
+    }
+  }
+  fb->capa = new_capa;
+}
+
+static VALUE
+fairy_fixnum_buffer_initialize(VALUE self)
+{
+  fairy_fixnum_buffer_t *fb;
+  GetFairyFixnumBufferPtr(self, fb);
+
+  fb->capa = FIXNUM_BUFFER_DEFAULT_CAPA;
+  fb->fixnums = ALLOC_N(unsigned char, fb->capa);
+  return self;
+}
+
+VALUE
+rb_fairy_fixnum_buffer_new(void)
+{
+  VALUE fb;
+  fb =  fairy_fixnum_buffer_alloc(rb_cFairyFixnumBuffer);
+  fairy_fixnum_buffer_initialize(fb);
+  return fb;
+}
+
+VALUE
+rb_fairy_fixnum_buffer_clear(VALUE self)
+{
+  fairy_fixnum_buffer_t *fb;
+  GetFairyFixnumBufferPtr(self, fb);
+
+  fb->capa = FIXNUM_BUFFER_DEFAULT_CAPA;
+  fb->length = 0;
+  fb->push = 0;
+  fb->pop = 0;
+  if (fb->fixnums) {
+    ruby_xfree(fb->fixnums);
+  }
+  fb->fixnums = ALLOC_N(unsigned char, fb->capa);
+  return self;
+}
+
+VALUE
+rb_fairy_fixnum_buffer_length(VALUE self)
+{
+  fairy_fixnum_buffer_t *fb;
+  
+  GetFairyFixnumBufferPtr(self, fb);
+  return LONG2NUM(fb->length);
+}
+
+
+VALUE
+rb_fairy_fixnum_buffer_realsize(VALUE self)
+{
+  fairy_fixnum_buffer_t *fb;
+  
+  GetFairyFixnumBufferPtr(self, fb);
+  return LONG2NUM(fb->push - fb->pop);
+}
+
+inline static int
+push_index(fairy_fixnum_buffer_t *fb)
+{
+  int idx = 0;
+  retry:
+  if (fb->push < fb->capa) {
+    idx = fb->push;
+  }
+  else if (fb->push - fb->capa < fb->pop) {
+    idx = fb->push - fb->capa;
+  }
+  else {
+    fairy_fixnum_buffer_resize_double_capa(fb);
+    goto retry;
+  }
+  return idx;
+}
+
+VALUE
+rb_fairy_fixnum_buffer_push_long(VALUE self, long v)
+{
+  fairy_fixnum_buffer_t *fb;
+  int idx, i;
+  char low;
+  GetFairyFixnumBufferPtr(self, fb);
+  
+  fb->length++;
+  for (i = 0; i <= (int)sizeof(long); i++) {
+    idx = push_index(fb);
+    low = (char)(v & 0x7f); 
+    v >>= 7;
+    if (v == 0 && low != 0x7f) {
+      fb->fixnums[idx] = low;
+      fb->push++;
+      return self;
+    }
+    else if (v == -1) {
+      low |= 0x80;
+      fb->fixnums[idx] = low;
+      fb->push++;
+      idx = push_index(fb);
+      fb->fixnums[idx] = 0x7f;
+      fb->push++;
+      return self;
+    }
+    else { 
+      low |= 0x80; 
+      fb->fixnums[idx] = low;
+      fb->push++;
+    } 
+  }
+  return self;
+}
+
+
+VALUE
+rb_fairy_fixnum_buffer_push(VALUE self, VALUE v)
+{
+  if (!FIXNUM_P(v)) {
+    rb_raise(rb_eTypeError, "not a fixnum");
+  }
+
+  return rb_fairy_fixnum_buffer_push_long(self, FIX2LONG(v));
+}
+
+inline static long
+at_long(fairy_fixnum_buffer_t *fb, long idx, long *ret)
+{
+  long v = 0;
+  long idx2;
+  int i = 0;
+  char low;
+  
+  while(1) {
+    if (idx < fb->capa) {
+      idx2 = idx;
+    }
+    else {
+      idx2 = idx - fb-> capa;
+    }
+    low = fb->fixnums[idx2];
+    idx++;
+
+    if (low & 0x80) {
+      v |= (low & 0x7f) << i;
+    }
+    else if (low == 0x7f) {
+      while (i <= (int)sizeof(long)*8) {
+	v |= 0x7f << i;
+	i += 7;
+      }
+      break;
+    }
+    else {
+      v |= (low & 0x7f) << i;
+      break;
+    }
+    i += 7;
+  }
+  if (ret != NULL) {
+    *ret = idx;
+  }
+  return v;
+}
+
+long
+rb_fairy_fixnum_buffer_pop_long(VALUE self)
+{
+  fairy_fixnum_buffer_t *fb;
+  long v = 0;
+  int i;
+  char low;
+  
+  GetFairyFixnumBufferPtr(self, fb);
+
+  if (fb->push == fb->pop)
+    return 0;
+
+  fb->length--;
+  i = 0;
+  while(1) {
+    low = fb->fixnums[fb->pop++];
+    if(fb->pop >= fb->capa) {
+      fb->pop -= fb->capa;
+      fb->push -= fb->capa;
+    }
+
+    if (low & 0x80) {
+      v |= (low & 0x7f) << i;
+    }
+    else if (low == 0x7f) {
+      while (i <= (int)sizeof(long)*8) {
+	v |= 0x7f << i;
+	i += 7;
+      }
+      break;
+    }
+    else {
+      v |= (low & 0x7f) << i;
+      break;
+    }
+    i += 7;
+  }
+  return v;
+}
+
+VALUE
+rb_fairy_fixnum_buffer_pop(VALUE self)
+{
+  fairy_fixnum_buffer_t *fb;
+  GetFairyFixnumBufferPtr(self, fb);
+  
+  if (fb->push == fb->pop)
+    return Qnil;
+  return LONG2FIX(rb_fairy_fixnum_buffer_pop_long(self));
+}
+
+VALUE
+rb_fairy_fixnum_buffer_each(VALUE self)
+{
+  fairy_fixnum_buffer_t *fb;
+  long i;
+  
+  GetFairyFixnumBufferPtr(self, fb);
+  
+  i = fb->pop;
+  while (i < fb->push) {
+    rb_yield(LONG2FIX(at_long(fb, i, &i)));
+  }
+  return self;
+}
+
+VALUE
+rb_fairy_fixnum_buffer_each_callback(VALUE self, VALUE(*callback)(long, VALUE), VALUE arg)
+{
+  fairy_fixnum_buffer_t *fb;
+  long i;
+  
+  GetFairyFixnumBufferPtr(self, fb);
+  
+  i = fb->pop;
+  while (i < fb->push) {
+    callback(at_long(fb, i, &i), arg);
+  }
+  return self;
+} 
+
+
+VALUE
+rb_fairy_fixnum_buffer_to_a(VALUE self)
+{
+  VALUE ary;
+  fairy_fixnum_buffer_t *fb;
+  VALUE s, v;
+  long i;
+  
+  GetFairyFixnumBufferPtr(self, fb);
+
+  ary = rb_ary_new2(fb->length);
+
+  i = fb->pop;
+  while (i < fb->push) {
+    v = LONG2FIX(at_long(fb, i, &i));
+    rb_ary_push(ary, v);
+  }
+  return ary;
+}
+
+VALUE
+rb_fairy_fixnum_buffer_marshal_dump(VALUE self)
+{
+  fairy_fixnum_buffer_t *fb;
+  long len;
+  VALUE str;
+
+  GetFairyFixnumBufferPtr(self, fb);
+
+  if (fb->push < fb->capa) {
+    str = rb_str_new(fb->fixnums + fb->pop, fb->push - fb->pop);
+  }
+  else {
+    str = rb_str_new(fb->fixnums + fb->pop, fb->capa - fb->pop);
+    rb_str_cat(str, fb->fixnums, fb->push - fb->capa);
+  }
+  return rb_ary_new3(3, LONG2NUM(fb->push - fb->pop), str, LONG2NUM(fb->length));
+}
+
+VALUE
+rb_fairy_fixnum_buffer_marshal_load(VALUE self, VALUE obj)
+{
+  fairy_fixnum_buffer_t *fb;
+  VALUE str;
+  GetFairyFixnumBufferPtr(self, fb);
+
+  fb->push = NUM2LONG(rb_ary_entry(obj, 0));
+
+  fb->capa = FIXNUM_BUFFER_DEFAULT_CAPA;
+  if (fb->capa <= fb->push) {
+    fb->capa += fb->push;
+  }
+
+  fb->fixnums = ALLOC_N(unsigned char, fb->capa);
+  MEMCPY(fb->fixnums, RSTRING_PTR(rb_ary_entry(obj, 1)),
+	 unsigned char, fb->push);
+
+  fb->length = NUM2LONG(rb_ary_entry(obj, 2));
+  
+  return self;
+}
+
+
+VALUE
+rb_fairy_fixnum_buffer_inspect(VALUE self)
+{
+  fairy_fixnum_buffer_t *fb;
+  VALUE str;
+  GetFairyFixnumBufferPtr(self, fb);
+
+  str = rb_sprintf("<%s:%x length=%d ", rb_obj_classname(self), (void*)self, fb->length);
+
+  rb_str_append(str, rb_inspect(rb_fairy_fixnum_buffer_to_a(self)));
+  rb_str_cat2(str, ">");
+  return str;
+}
+    
+VALUE
+rb_fairy_fixnum_buffer_inspect_raw(VALUE self)
+{
+  fairy_fixnum_buffer_t *fb;
+  VALUE str;
+  GetFairyFixnumBufferPtr(self, fb);
+
+  str = rb_sprintf("<%s:%x length=%d: [", rb_obj_classname(self), (void*)self, fb->length);
+  if (fb->push < fb->capa) {
+    VALUE s;
+    long i;
+    
+    for (i = fb->pop; i < fb->push; i++) {
+      int b = fb->fixnums[i] >> 7;
+      char v = fb->fixnums[i] & 0x7f;
+      
+      s = rb_sprintf(" %d:%d", b, v);
+      rb_str_append(str, s);
+    }
+  }
+  else {
+    VALUE s;
+    long i;
+    for (i = fb->pop; i < fb->capa; i++) {
+      int b = fb->fixnums[i] >> 7;
+      char v = fb->fixnums[i] & 0x7f;
+      
+      s = rb_sprintf(" %d:%d", b, v);
+      rb_str_append(str, s);
+    }
+
+    for (i = 0; i < fb->push - fb->capa; i++) {
+      int b = fb->fixnums[i] >> 7;
+      char v = fb->fixnums[i] & 0x7f;
+      
+      s = rb_sprintf(" %d:%d", b, v);
+      rb_str_append(str, s);
+    }
+  }
+  rb_str_cat2(str, "]>");
+  return str;
+}
+
+void
+Init_fixnum_buffer()
+{
+  VALUE ffb;
+  
+  rb_cFairyFixnumBuffer = rb_define_class_under(rb_mFairy, "FixnumBuffer", rb_cObject);
+
+  ffb = rb_cFairyFixnumBuffer;
+  rb_define_alloc_func(ffb, fairy_fixnum_buffer_alloc);
+  rb_define_method(ffb, "initialize", fairy_fixnum_buffer_initialize, 0);
+  rb_define_method(ffb, "length", rb_fairy_fixnum_buffer_length, 0);
+  rb_define_alias(ffb,  "size", "length");
+  rb_define_method(ffb, "realsize", rb_fairy_fixnum_buffer_realsize, 0);
+  rb_define_method(ffb, "push", rb_fairy_fixnum_buffer_push, 1);
+  rb_define_method(ffb, "pop", rb_fairy_fixnum_buffer_pop, 0);
+  rb_define_method(ffb, "each", rb_fairy_fixnum_buffer_each, 0);
+  rb_define_method(ffb, "to_a", rb_fairy_fixnum_buffer_to_a, 0);
+  rb_define_method(ffb, "marshal_dump", rb_fairy_fixnum_buffer_marshal_dump, 0);
+  rb_define_method(ffb, "marshal_load", rb_fairy_fixnum_buffer_marshal_load, 1);
+  rb_define_method(ffb, "inspect", rb_fairy_fixnum_buffer_inspect, 0);
+}