tx 0.0.5

data/ext/ssv.hpp

@@ -0,0 +1,93 @@
+#ifndef __SSV_HPP__
+#define __SSV_HPP__
+
+#include <memory.h>
+#include <vector>
+#include <cassert>
+#include <stdio.h>
+
+namespace tx_tool{
+
+  typedef unsigned int uint; // 32bit
+  typedef unsigned short ushort; // 16bit
+  typedef unsigned char uchar; // 8bit
+
+
+#define SSV_BLOCK_SHIFT  (5)
+#define SSV_BLOCK        (1U << SSV_BLOCK_SHIFT)
+
+#define SSV_LBLOCK_SHIFT (8)
+#define SSV_LBLOCK		 (1U << SSV_LBLOCK_SHIFT)
+#define SSV_MBLOCK_SHIFT (5)
+#define SSV_MBLOCK       (1U << SSV_MBLOCK_SHIFT)
+
+#define logLL (16)
+#define LL (1U << logLL)
+#define logLLL (5)
+#define LLL (1U << logLLL)
+#define logL (logLL-1-5)
+#define L (1U << logL)
+
+  class ssv{
+
+  public:
+    ssv(const uint _size = 0);
+    ssv(std::vector<bool>& bv);
+
+    int resize(const uint _size);
+
+    void free();
+
+    ~ssv();
+
+    inline uint getBit(const uint pos) const {
+      return (B[pos / SSV_BLOCK] >> (pos % SSV_BLOCK)) & 1;
+    }
+
+    uint getBits(const uint pos, uint width) const;
+    void setBits(const uint pos, const uint width, const uint x);
+    uint rank(uint pos, const uint bit) const;
+    uint select(uint pos, const uint bit) const;
+
+    void setBit(uint pos, uint x);
+    uint getAllocate() const;
+
+    void build();
+
+    uint rankBuild(const uint t_size); // return oneNum
+    void selectBuild(const uint t_size);
+
+    int write(FILE* outfp);
+    int read(FILE* infp);
+
+    uint getBlock(const uint blockPos) const;
+    void setBlock(const uint blockPos, const uint x);
+    uint getSize() const;
+    uint getBlockSize() const;
+
+    size_t set_array(void* ptr);
+
+  private:
+    uint popCount(uint r) const;
+    uint _rank1(const uint pos) const;
+    uint _select1(uint x) const;
+    uint _select0(uint x) const;
+    uint* B;
+    uint size;
+    uint oneNum;
+    uint blockSize;	// (size+SSV_BLOCK-1)/SSV_BLOCKSZIE
+
+    uint LBlockSize;
+    uint MBlockSize;
+
+    // for rank
+    uint* levelL;
+    uchar* levelM;
+
+    bool isBuild;
+    bool no_delete;
+  };
+
+}
+
+#endif

data/ext/swig.patch

@@ -0,0 +1,192 @@
+Index: Lib/ruby/rubycontainer.swg
+===================================================================
+--- Lib/ruby/rubycontainer.swg	(revision 11423)
++++ Lib/ruby/rubycontainer.swg	(working copy)
+@@ -446,11 +446,10 @@
+   %typemap(out,noblock=1,fragment="RubySequence_Cont")
+     std::pair<const_iterator, const_iterator> {
+     $result = rb_ary_new2(2);
+-    RARRAY_PTR($result)[0] = SWIG_NewPointerObj(swig::make_const_iterator(%static_cast($1,const $type &).first),
+-						swig::ConstIterator::descriptor(),SWIG_POINTER_OWN);
+-    RARRAY_PTR($result)[1] = SWIG_NewPointerObj(swig::make_const_iterator(%static_cast($1,const $type &).second),
+-						swig::ConstIterator::descriptor(),SWIG_POINTER_OWN);
+-    RARRAY_LEN($result) = 2;
++    rb_ary_push($result, SWIG_NewPointerObj(swig::make_const_iterator(%static_cast($1,const $type &).first), 
++	    swig::ConstIterator::descriptor(),SWIG_POINTER_OWN)); 
++    rb_ary_push($result, SWIG_NewPointerObj(swig::make_const_iterator(%static_cast($1,const $type &).second), 
++	    swig::ConstIterator::descriptor(),SWIG_POINTER_OWN)); 
+   }
+ 
+   // std::map/multimap/set allow returning std::pair< iterator, iterator > from
+@@ -459,11 +458,10 @@
+   %typemap(out,noblock=1,fragment="RubySequence_Cont")
+     std::pair<iterator, iterator> {
+     $result = rb_ary_new2(2);
+-    RARRAY_PTR($result)[0] = SWIG_NewPointerObj(swig::make_const_iterator(%static_cast($1,const $type &).first),
+-						swig::ConstIterator::descriptor(),SWIG_POINTER_OWN);
+-    RARRAY_PTR($result)[1] = SWIG_NewPointerObj(swig::make_const_iterator(%static_cast($1,const $type &).second),
+-						swig::ConstIterator::descriptor(),SWIG_POINTER_OWN);
+-    RARRAY_LEN($result) = 2;
++    rb_ary_push($result, SWIG_NewPointerObj(swig::make_const_iterator(%static_cast($1,const $type &).first), 
++	    swig::ConstIterator::descriptor(),SWIG_POINTER_OWN)); 
++    rb_ary_push($result, SWIG_NewPointerObj(swig::make_const_iterator(%static_cast($1,const $type &).second), 
++	    swig::ConstIterator::descriptor(),SWIG_POINTER_OWN)); 
+   }
+ 
+ 
+@@ -566,7 +564,8 @@
+     {
+       Sequence::const_iterator i = $self->begin();
+       Sequence::const_iterator e = $self->end();
+-      VALUE str = rb_str_new2( swig::type_name< Sequence >() );
++      // Double parentheses to avoid macro expansion error in case Sequence contains comma.
++      VALUE str = rb_str_new2(( swig::type_name< Sequence >() ));
+       str = rb_str_cat2( str, " [" );
+       bool comma = false;
+       VALUE tmp;
+@@ -1112,9 +1111,8 @@
+ 	int i = 0;
+ 	for (const_iterator it = seq.begin();
+ 	     it != seq.end(); ++it, ++i) {
+-	  RARRAY_PTR(obj)[i] = swig::from< value_type >(*it);
++	  rb_ary_push(obj, swig::from< value_type >(*it)); 
+ 	}
+-	RARRAY_LEN(obj) = size;
+ 	rb_obj_freeze(obj);  // treat as immutable result
+ 	return obj;
+       } else {
+Index: Lib/ruby/std_set.i
+===================================================================
+--- Lib/ruby/std_set.i	(revision 11423)
++++ Lib/ruby/std_set.i	(working copy)
+@@ -170,10 +170,9 @@
+   %typemap(out,noblock=1,fragment="RubyPairBoolOutputIterator")
+   std::pair<iterator, bool> {
+     $result = rb_ary_new2(2);
+-    RARRAY_PTR($result)[0] = SWIG_NewPointerObj(swig::make_set_nonconst_iterator(%static_cast($1,$type &).first),
+-						swig::Iterator::descriptor(),SWIG_POINTER_OWN);    
+-    RARRAY_PTR($result)[1] = SWIG_From(bool)(%static_cast($1,const $type &).second);
+-    RARRAY_LEN($result) = 2;
++    rb_ary_push($result, SWIG_NewPointerObj(swig::make_set_nonconst_iterator(%static_cast($1,$type &).first), 
++                                            swig::Iterator::descriptor(),SWIG_POINTER_OWN)); 
++    rb_ary_push($result, SWIG_From(bool)(%static_cast($1,const $type &).second)); 
+    }
+ 
+   %extend  {
+Index: Lib/ruby/std_multimap.i
+===================================================================
+--- Lib/ruby/std_multimap.i	(revision 11423)
++++ Lib/ruby/std_multimap.i	(working copy)
+@@ -115,7 +115,8 @@
+     {
+       MultiMap::iterator i = $self->begin();
+       MultiMap::iterator e = $self->end();
+-      VALUE str = rb_str_new2( swig::type_name< MultiMap >() );
++      // Double parentheses to avoid macro expansion error in case MultiMap contains comma.
++      VALUE str = rb_str_new2(( swig::type_name< MultiMap >() ));
+       str = rb_str_cat2( str, " {" );
+       VALUE tmp;
+       while ( i != e )
+Index: Lib/ruby/std_pair.i
+===================================================================
+--- Lib/ruby/std_pair.i	(revision 11423)
++++ Lib/ruby/std_pair.i	(working copy)
+@@ -118,11 +118,9 @@
+ 
+       static VALUE from(const std::pair<T,U>& val) {
+ 	VALUE obj = rb_ary_new2(2);
+-	RARRAY_PTR(obj)[0] = swig::from< 
+-	  typename swig::noconst_traits<T >::noconst_type>(val.first);
+-	RARRAY_PTR(obj)[1] = swig::from(val.second);
+-	RARRAY_LEN(obj) = 2;
+-	rb_define_singleton_method(obj, "second",  
++	rb_ary_push(obj, swig::from<typename swig::noconst_traits<T >::noconst_type>(val.first)); 
++	rb_ary_push(obj, swig::from(val.second)); 
++	rb_define_singleton_method(obj, "second", 
+ 				   VALUEFUNC(_wrap_pair_second), 0 );
+ 	rb_define_singleton_method(obj, "second=", 
+ 				   VALUEFUNC(_wrap_pair_second_eq), 1 );
+@@ -148,7 +146,8 @@
+   VALUE inspect() const
+     {
+       VALUE tmp;
+-      VALUE str = rb_str_new2( swig::type_name< pair >() );
++      // Double parentheses to avoid macro expansion error in case pair contains comma.
++      VALUE str = rb_str_new2(( swig::type_name< pair >() ));
+       str = rb_str_cat2( str, " (" );
+       tmp = swig::from( $self->first );
+       tmp = rb_obj_as_string( tmp );
+Index: Lib/ruby/rubyclasses.swg
+===================================================================
+--- Lib/ruby/rubyclasses.swg	(revision 11423)
++++ Lib/ruby/rubyclasses.swg	(working copy)
+@@ -315,30 +315,36 @@
+ 
+   };
+ 
+-  ID  GC_VALUE::hash_id = rb_intern("hash");
+-  ID  GC_VALUE::lt_id = rb_intern("<");
+-  ID  GC_VALUE::gt_id = rb_intern(">");
+-  ID  GC_VALUE::eq_id = rb_intern("==");
+-  ID  GC_VALUE::le_id = rb_intern("<=");
+-  ID  GC_VALUE::ge_id = rb_intern(">=");
++  // We need this because rb_intern macro uses statement-expression and
++  // statement-expression is allowed only inside functions.
++  static ID rb_intern_wrapper(const char* str) {
++    return rb_intern(str);
++  }
+ 
+-  ID  GC_VALUE::pos_id = rb_intern("+@");
+-  ID  GC_VALUE::neg_id = rb_intern("-@");
+-  ID  GC_VALUE::inv_id = rb_intern("~");
++  ID  GC_VALUE::hash_id = rb_intern_wrapper("hash");
++  ID  GC_VALUE::lt_id = rb_intern_wrapper("<");
++  ID  GC_VALUE::gt_id = rb_intern_wrapper(">");
++  ID  GC_VALUE::eq_id = rb_intern_wrapper("==");
++  ID  GC_VALUE::le_id = rb_intern_wrapper("<=");
++  ID  GC_VALUE::ge_id = rb_intern_wrapper(">=");
+ 
+-  ID  GC_VALUE::add_id = rb_intern("+");
+-  ID  GC_VALUE::sub_id = rb_intern("-");
+-  ID  GC_VALUE::mul_id = rb_intern("*");
+-  ID  GC_VALUE::div_id = rb_intern("/");
+-  ID  GC_VALUE::mod_id = rb_intern("%");
++  ID  GC_VALUE::pos_id = rb_intern_wrapper("+@");
++  ID  GC_VALUE::neg_id = rb_intern_wrapper("-@");
++  ID  GC_VALUE::inv_id = rb_intern_wrapper("~");
+ 
+-  ID  GC_VALUE::and_id = rb_intern("&");
+-  ID  GC_VALUE::or_id  = rb_intern("|");
+-  ID  GC_VALUE::xor_id = rb_intern("^");
++  ID  GC_VALUE::add_id = rb_intern_wrapper("+");
++  ID  GC_VALUE::sub_id = rb_intern_wrapper("-");
++  ID  GC_VALUE::mul_id = rb_intern_wrapper("*");
++  ID  GC_VALUE::div_id = rb_intern_wrapper("/");
++  ID  GC_VALUE::mod_id = rb_intern_wrapper("%");
+ 
+-  ID  GC_VALUE::lshift_id = rb_intern("<<");
+-  ID  GC_VALUE::rshift_id = rb_intern(">>");
++  ID  GC_VALUE::and_id = rb_intern_wrapper("&");
++  ID  GC_VALUE::or_id  = rb_intern_wrapper("|");
++  ID  GC_VALUE::xor_id = rb_intern_wrapper("^");
+ 
++  ID  GC_VALUE::lshift_id = rb_intern_wrapper("<<");
++  ID  GC_VALUE::rshift_id = rb_intern_wrapper(">>");
++
+   VALUE GC_VALUE::_hash = Qnil;
+ 
+   typedef GC_VALUE LANGUAGE_OBJ;
+Index: Lib/ruby/std_map.i
+===================================================================
+--- Lib/ruby/std_map.i	(revision 11423)
++++ Lib/ruby/std_map.i	(working copy)
+@@ -345,7 +345,8 @@
+     {
+       Map::const_iterator i = $self->begin();
+       Map::const_iterator e = $self->end();
+-      VALUE str = rb_str_new2( swig::type_name< Map >() );
++      // Double parentheses to avoid macro expansion error in case Map contains comma.
++      VALUE str = rb_str_new2(( swig::type_name< Map >() ));
+       str = rb_str_cat2( str, " {" );
+       bool comma = false;
+       VALUE tmp;

data/ext/tx.cpp

@@ -0,0 +1,442 @@
+#include "tx.hpp"
+
+#include <climits>
+#include "ssv.hpp"
+namespace tx_tool{
+
+  uint tx::NOTFOUND = UINT_MAX;
+
+  struct queue_elem{
+    queue_elem(size_t _left, size_t _right, int _depth) :left(_left),right(_right),depth(_depth){}
+    size_t left;
+    size_t right;
+    int depth;
+  };
+
+  tx::tx():edge(NULL), keyNum(0), no_delete(false) {}
+
+  tx::~tx(){
+    if (!no_delete) {
+      delete[] edge;
+      edge = NULL;
+      no_delete = false;
+    }
+  }
+
+  int tx::build(std::vector<std::string>& wordList, const char* fileName) {
+    sort(wordList.begin(),wordList.end());
+    const size_t origWordNum = wordList.size();
+    wordList.erase(unique(wordList.begin(),wordList.end()),wordList.end());
+    int keyNum = (int)wordList.size();
+    if (keyNum != origWordNum){
+      resultLog << "shrink word list " << origWordNum << " -> " << keyNum << std::endl;
+    } else {
+      resultLog << "word list " << keyNum << " elements" << std::endl;
+    }
+
+    uint totalSize = 0;
+    for (size_t i = 0; i < wordList.size(); i++){
+      totalSize += (uint)wordList[i].size();
+    }
+                
+    FILE* outfp = fopen(fileName,"wb");
+    if (outfp == NULL){
+      errorLog << "cannot open " << fileName << std::endl;
+      return -1;
+    }
+        
+    std::queue<queue_elem> q;
+    if (keyNum != 0){
+      q.push(queue_elem(0,keyNum,0));
+    }
+    if (fwrite(&keyNum,sizeof(int),1,outfp) != 1){
+      errorLog << "fwrite error " << std::endl;
+      return -1;
+    }
+
+    std::vector<bool> vb_loud;
+    std::vector<bool> vb_terminal;
+
+    vb_loud.push_back(0); // super root
+    vb_loud.push_back(1); 
+
+    uint nodeNum = 0;
+
+    while (!q.empty()){
+      queue_elem& elem = q.front();
+      const int depth    = elem.depth;
+      const size_t left  = elem.left;
+      const size_t right = elem.right;
+      q.pop();
+
+      nodeNum++;
+      size_t newLeft = left;
+      if (wordList[left].size() == depth){
+        vb_terminal.push_back(1); // this node has terminate
+        newLeft++;
+        if (newLeft == right){
+          vb_loud.push_back(1);
+          continue;
+        }
+      } else {
+        vb_terminal.push_back(0);
+      }
+      size_t prev = newLeft;
+      char prev_c = wordList[prev][depth];
+      for (size_t i = newLeft+1; i < right; i++){
+        if (prev_c != wordList[i][depth]){
+          fputc(prev_c,outfp);
+          vb_loud.push_back(0);
+          q.push(queue_elem(prev,i,depth+1));
+          prev = i;
+          prev_c = wordList[prev][depth];
+        }
+      }
+      if (prev != right){
+        fputc(prev_c,outfp);
+        vb_loud.push_back(0);
+        q.push(queue_elem(prev,right,depth+1));
+      }
+      vb_loud.push_back(1);
+    }
+
+    {
+      ssv sv(vb_loud);
+      sv.build();
+      if (sv.write(outfp) == -1){
+        errorLog << "fwrite error " << std::endl;
+        return -1;
+      }
+    }
+    {
+      ssv sv(vb_terminal);
+      sv.build();
+      if (sv.write(outfp) == -1){
+        errorLog << "fwrite error " << std::endl;
+        return -1;
+      }
+    }
+        
+    if (fwrite(&nodeNum,sizeof(int),1,outfp) != 1){
+      errorLog << "fwrite error " << std::endl;
+      return -1;
+    }
+
+    size_t outfpSize = ftell(outfp);
+    resultLog << "outputSize:" << outfpSize << " inputSize:" << totalSize << " ratio:" << (float)outfpSize/totalSize << std::endl;
+    if (outfp) fclose(outfp);
+    return 0;
+  }
+
+  int tx::read(const char* fileName){
+    FILE* infp = fopen(fileName,"rb");
+    if (infp == NULL){
+      errorLog << "cannot open " << fileName << std::endl;
+      return -1;
+    }
+
+    keyNum = 0;
+    if (fread(&keyNum,sizeof(int),1,infp) != 1){
+      errorLog << "keyNum read error" << std::endl;
+      fclose(infp);
+      return -1;
+    }
+
+    fseek(infp,0,SEEK_END);
+    size_t fileSize = ftell(infp);
+    if (fseek(infp,fileSize-(1*sizeof(int)),SEEK_SET) == -1){
+      errorLog << "fseek error" << std::endl;
+      fclose(infp);
+      return -1;
+    }
+
+    int nodeNum = -1;
+    if (fread(&nodeNum,sizeof(int),1,infp) != 1){
+      errorLog << "nodeNum read error" << std::endl;
+      fclose(infp);
+      return -1;
+    }
+    resultLog << "keyNum:" << (int)keyNum << " nodeNum:" << nodeNum << std::endl;
+
+    if (fseek(infp,sizeof(int)*1,SEEK_SET) == -1){
+      errorLog << "fseek error" << std::endl;
+      fclose(infp);
+      return -1;
+    }
+
+    if (nodeNum > 0){
+      edge = new char [nodeNum-1];
+      if (fread(edge,sizeof(char),nodeNum-1,infp) != nodeNum-1){
+        errorLog << "fseek error" << std::endl;
+        fclose(infp);
+        return -1;
+      }
+    }
+
+    loud.read(infp);
+    terminal.read(infp);
+
+    if (infp) fclose(infp);
+    return 0;
+  }
+
+  int tx::setArray(void* ptr, size_t readSize){
+    keyNum = *(uint*)(ptr);
+    printf("keyNum:%d\n", keyNum);
+    int nodeNum = *(uint*)((uchar*)ptr+readSize-sizeof(uint));
+    printf("nodeNum:%d\n", nodeNum);
+    edge = (char*)ptr + sizeof(uint);
+    size_t readNum = loud.set_array((void*)((uchar*)ptr + sizeof(uint) + nodeNum - 1));
+    size_t readNum2 = terminal.set_array((void*)((uchar*)ptr + sizeof(uint) + nodeNum - 1 + readNum));
+    if (sizeof(uint) + nodeNum + - 1 + readNum + readNum2 + sizeof(uint) != readSize){
+      errorLog << "setArray error" << std::endl;
+      return -1;
+    }
+    no_delete = true;
+    return 0;
+  }
+  
+  uint tx::prefixSearch(const char* str, const size_t len, size_t& retLen, bool matchPrefix) const {
+    uint curPos = 2;
+    uint retId = NOTFOUND;
+    retLen = NOTFOUND;
+    if (terminal.getSize() <= 2) return retId;
+
+    for (size_t i = 0 ; ; i++){
+      const uint nodeId = loud.rank(curPos-1,1)-1;
+      if (terminal.getBit(nodeId)){
+        retLen = i;
+        retId = terminal.rank(nodeId,1)-1;
+      } else if (matchPrefix){
+        retLen = i;
+        retId = NOTFOUND;
+      }
+      if (i == len) break;
+      uint nextPos = getChild(curPos,str[i]);
+      if (nextPos == UINT_MAX){
+        break;
+      }
+      curPos = nextPos;
+    }
+    return retId;
+  }
+
+  uint tx::expandSearch(const char* str, const size_t len, std::vector<std::string>& ret, const uint limit) const {
+    ret.clear();
+    if (limit == 0) return 0;
+    if (terminal.getSize() <= 2) return 0;
+
+    bool prefix = false;
+    uint curPos = 2;
+    for (size_t i = 0; i < len; i++){
+      uint nextPos = getChild(curPos,str[i]);      
+      const uint nodeId = loud.rank(curPos-1,1)-1;
+      if (terminal.getBit(nodeId)){
+        ret.push_back(std::string(str,str+i));
+      }
+      
+      if (nextPos == UINT_MAX){
+        prefix = true;
+        break;
+      }
+      curPos = nextPos;
+    }
+    
+    if (!prefix){
+      std::string curStr(str, len);
+      std::vector<std::pair<size_t, std::pair<std::string, uint> > > ret_p;
+      enumerateAll(curPos,curStr,ret_p); 
+      sort(ret_p.begin(),ret_p.end());
+      for (size_t i = 0; i < ret_p.size() && i < limit; i++){
+        ret.push_back(ret_p[i].second.first);
+      }
+    } 
+    return (uint)ret.size();
+  }
+
+  uint tx::commonPrefixSearch(const char* str, const size_t len, std::vector<std::string>& ret, std::vector<uint>& retID, const uint limit) const{
+    ret.clear();
+    retID.clear();
+    if (limit == 0) return 0;
+    if (terminal.getSize() <= 2) return 0;
+
+    uint curPos = 2;
+
+    for (size_t i = 0; ; i++){
+      const uint nodeId = loud.rank(curPos-1,1)-1;
+      if (terminal.getBit(nodeId)){
+        ret.push_back(std::string(str, str+i));
+        retID.push_back(terminal.rank(nodeId,1)-1);
+        if (ret.size() == limit) break;
+      }
+      if (i == len) break;
+      
+      uint nextPos = getChild(curPos,str[i]);      
+      if (nextPos == UINT_MAX){
+        break;
+      }
+      curPos = nextPos;
+    }    
+    return (uint)ret.size();
+  }
+
+  uint tx::commonPrefixSearch(const char* str, const size_t len, std::vector<uint>& retLen, std::vector<uint>& retID, const uint limit) const{
+    retLen.clear();
+    retID.clear();
+    if (limit == 0) return 0;
+    if (terminal.getSize() <= 2) return 0;
+
+    uint curPos = 2;
+    for (size_t i = 0; ; i++){
+      const uint nodeId = loud.rank(curPos-1,1)-1;
+      if (terminal.getBit(nodeId)){
+        retLen.push_back(i);
+        retID.push_back(terminal.rank(nodeId,1)-1);
+        if (retLen.size() == limit) break;
+      }
+      if (i == len) break;
+      
+      uint nextPos = getChild(curPos,str[i]);      
+      if (nextPos == UINT_MAX){
+        break;
+      }
+      curPos = nextPos;
+    }
+    
+    return (uint)retLen.size();
+  }
+
+  
+  uint tx::predictiveSearch(const char* str, const size_t len, std::vector<std::string>& ret, std::vector<uint>& retID, const uint limit) const{
+    ret.clear();
+    retID.clear();
+    if (limit == 0) return 0;
+    if (terminal.getSize() <= 2) return 0;
+    
+    bool prefix = false;
+    uint curPos = 2;
+    for (size_t i = 0; i < len; i++){
+      uint nextPos = getChild(curPos,str[i]);      
+      if (nextPos == UINT_MAX){
+        prefix = true;
+        break;
+      }
+      curPos = nextPos;
+    }
+    
+    if (!prefix){
+      std::string curStr(str, len);
+      std::vector<std::pair<size_t, std::pair<std::string, uint> > > ret_p;
+      enumerateAll(curPos, curStr, ret_p); 
+      sort(ret_p.begin(),ret_p.end());
+      for (size_t i = 0; i < ret_p.size() && i < limit; i++){
+        ret.push_back(ret_p[i].second.first);
+        retID.push_back(ret_p[i].second.second);
+      }
+    } 
+    return (uint)ret.size();
+  }
+
+  uint tx::predictiveSearch(const char* str, const size_t len, std::vector<uint>& retLen, std::vector<uint>& retID, const uint limit) const{
+    retLen.clear();
+    retID.clear();
+    if (limit == 0) return 0;
+    if (terminal.getSize() <= 2) return 0;
+    
+    bool prefix = false;
+    uint curPos = 2;
+    for (size_t i = 0; i < len; i++){
+      uint nextPos = getChild(curPos,str[i]);      
+      if (nextPos == UINT_MAX){
+        prefix = true;
+        break;
+      }
+      curPos = nextPos;
+    }
+    
+    if (!prefix){
+      std::string curStr(str, len);
+      std::vector<std::pair<size_t, std::pair<std::string, uint> > > ret_p;
+      enumerateAll(curPos, curStr, ret_p); 
+      sort(ret_p.begin(),ret_p.end());
+      for (size_t i = 0; i < ret_p.size() && i < limit; i++){
+        retLen.push_back(ret_p[i].second.first.size());
+        retID.push_back(ret_p[i].second.second);
+      }
+    } 
+    return (uint)retLen.size();
+  }
+
+
+
+  void tx::enumerateAll(const uint pos, const std::string str, std::vector<std::pair<size_t, std::pair<std::string, uint> > >& ret) const{
+    const uint nodeId = loud.rank(pos-1,1)-1;
+    if (terminal.getBit(nodeId)){
+      std::pair<std::string, uint> tmp(str,  terminal.rank(nodeId,1)-1);
+      ret.push_back(std::make_pair<size_t, std::pair<std::string, uint> >(str.size(), tmp));
+    }
+
+    uint curPos = pos;
+    uint edgePos = loud.rank(pos,0)-2;
+    while (loud.getBit(curPos) == 0){
+      const uint nextPos = loud.select(loud.rank(curPos,0),1)+1;
+      enumerateAll(nextPos,str + edge[edgePos],ret);
+      curPos++;
+      edgePos++;
+    }
+  }
+
+  uint tx::getChild(const uint pos, const char c) const{
+    uint curPos = pos;
+    uint edgePos = loud.rank(pos,0)-2;
+    for (;;){
+      if (loud.getBit(curPos) == 1) {
+        curPos = UINT_MAX;
+        return curPos;
+      }
+      if (edge[edgePos] == c){
+        uint nextPos = loud.select(loud.rank(curPos,0),1)+1;
+        return nextPos;
+      }
+      curPos++;
+      edgePos++;
+    }
+  }
+
+  uint tx::getParent(const uint pos, char& c) const{
+    c = edge[loud.rank(pos,0)-2];
+    return loud.select(loud.rank(pos-1, 1), 0);
+  }
+
+
+  uint tx::reverseLookup(const uint id, std::string& ret) const {
+    ret.clear();
+    if (id >= keyNum) return 0;
+    if (terminal.getSize() <= 2) return 0;
+
+    const uint nodeId = terminal.select(id + 1, 1);
+    char unused_c = 0;
+    uint curPos = getParent(loud.select(nodeId+1,1)+1, unused_c);
+    while (curPos >= 2){
+      char c = 0;
+      curPos = getParent(curPos, c);
+      ret += c;
+    }
+    reverse(ret.begin(), ret.end());
+    return ret.size();
+  }
+
+  std::string tx::getResultLog() const {
+    return resultLog.str();
+  }
+
+  std::string tx::getErrorLog() const{
+    return errorLog.str();
+  }
+
+  uint tx::getKeyNum() const {
+    return keyNum;
+  }
+
+
+} // namespace tx_tool