text_ux 0.0.1

data/vendor/ux-trie/ux-0.1.9/src/rsDic.cpp

@@ -0,0 +1,121 @@
+/* 
+ *  Copyright (c) 2010 Daisuke Okanohara
+ * 
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ * 
+ *   1. Redistributions of source code must retain the above Copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above Copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ *   3. Neither the name of the authors nor the names of its contributors
+ *      may be used to endorse or promote products derived from this
+ *      software without specific prior written permission.
+ */
+
+#include <iostream>
+#include <cassert>
+#include "rsDic.hpp"
+
+using namespace std;
+
+namespace ux {
+
+RSDic::RSDic() : size_(0) {
+}
+
+RSDic::~RSDic() {
+}
+
+void RSDic::build(BitVec& bv){
+  size_ = bv.size();
+  swap(bitVec_, bv);
+  L_.resize((size_ + L_BLOCK-1) / L_BLOCK);
+  size_t sum = 0;
+  for (uint64_t il = 0; il < size_; il += L_BLOCK){
+    L_[il/L_BLOCK] = sum;
+    for (uint64_t is = 0; is < L_BLOCK && il + is < size_; is += S_BLOCK){
+      sum += popCount(bitVec_.lookupBlock((il + is)/S_BLOCK));
+    }
+  }
+  L_.push_back(sum);
+}
+
+uint64_t RSDic::rank(const uint64_t pos, const uint8_t b) const{
+  uint64_t pos1  = pos+1;
+  uint64_t rank1 = L_[pos1 >> L_SHIFT];
+  uint64_t bpos  = (pos1 >> L_SHIFT) << (L_SHIFT - S_SHIFT);
+  uint64_t epos  = pos1 >> S_SHIFT; 
+  for (uint64_t i = bpos; i < epos; ++i){
+    rank1 += popCount(bitVec_.lookupBlock(i));
+  }
+  rank1 += popCountMasked(bitVec_.lookupBlock(epos), pos1 % S_BLOCK);
+  
+  if (b == 1) return rank1;
+  else        return pos1 - rank1;
+}
+
+uint64_t RSDic::select(const uint64_t pos, const uint8_t b) const{
+  uint64_t retPos = 0;
+  uint64_t posS   = selectOverL(pos, b, retPos);
+  return posS * S_BLOCK + selectBlock(retPos, bitVec_.lookupBlock(posS), b);
+}
+
+uint64_t RSDic::selectOverL(const uint64_t pos, const uint8_t b, uint64_t& retPos) const {
+  uint64_t left   = 0;
+  uint64_t right  = L_.size();
+  
+  retPos = pos;
+  while (left < right){
+    uint64_t mid = (left + right)/2;
+    assert(mid < L_.size());
+    if (getBitNum(L_[mid], L_BLOCK * mid, b) < retPos) left  = mid+1;
+    else                                               right = mid;
+  }
+  uint64_t posL = (left != 0) ? left - 1 : 0;
+  uint64_t posS  = posL * S_RATIO;
+
+  assert(retPos >= getBitNum(L_[posL], L_BLOCK * posL, b));
+
+  retPos -= getBitNum(L_[posL], L_BLOCK * posL, b);
+  for (;;posS++){
+    if (posS >= bitVec_.size()) break;
+    uint64_t num = getBitNum(popCount(bitVec_.lookupBlock(posS)), S_BLOCK, b);
+    if (retPos <= num) break;
+    retPos -= num;
+  }
+  return posS;
+}
+
+void RSDic::save(ostream& ofs) const{
+  bitVec_.save(ofs);
+}
+
+void RSDic::load(istream& ifs) {
+  bitVec_.load(ifs);
+  build(bitVec_);
+}
+
+size_t RSDic::getAllocSize() const {
+  return bitVec_.getAllocSize() + sizeof(L_[0]) * L_.size();
+}
+
+uint8_t RSDic::getBit(const uint64_t pos) const{
+  return bitVec_.getBit(pos);
+}
+
+size_t RSDic::size() const {
+  return bitVec_.size();
+}
+
+void RSDic::clear() {
+  bitVec_.clear();
+  L_.clear();
+  size_ = 0;
+}
+
+}

data/vendor/ux-trie/ux-0.1.9/src/rsDic.hpp

@@ -0,0 +1,57 @@
+/* 
+ *  Copyright (c) 2010 Daisuke Okanohara
+ * 
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ * 
+ *   1. Redistributions of source code must retain the above Copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above Copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ *   3. Neither the name of the authors nor the names of its contributors
+ *      may be used to endorse or promote products derived from this
+ *      software without specific prior written permission.
+ */
+
+#ifndef RSDIC_HPP__
+#define RSDIC_HPP__
+
+#include <stdint.h>
+#include <vector>
+#include <iostream>
+#include "bitVec.hpp"
+#include "uxUtil.hpp"
+
+namespace ux {
+
+class RSDic {
+public:
+  RSDic();
+  ~RSDic();
+
+  void build(BitVec& bv);
+  uint64_t rank(uint64_t pos, uint8_t b) const;
+  uint64_t select(uint64_t pos, uint8_t b) const;
+
+  void save(std::ostream& os) const;
+  void load(std::istream& is);
+  size_t getAllocSize() const;
+  uint8_t getBit(uint64_t pos) const;
+  size_t size() const;
+  void clear();
+
+private:
+  uint64_t selectOverL(uint64_t pos, uint8_t b, uint64_t& retPos) const;
+  
+  BitVec bitVec_;
+  std::vector<uint64_t> L_;
+  size_t size_;
+};
+
+}
+
+#endif // RSDIC_HPP__

data/vendor/ux-trie/ux-0.1.9/src/ux.hpp

@@ -0,0 +1,26 @@
+/* 
+ *  Copyright (c) 2010 Daisuke Okanohara
+  * 
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ * 
+ *   1. Redistributions of source code must retain the above Copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above Copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ *   3. Neither the name of the authors nor the names of its contributors
+ *      may be used to endorse or promote products derived from this
+ *      software without specific prior written permission.
+ */
+
+#ifndef UX_HPP__
+#define UX_HPP__
+
+#include "uxTrie.hpp"
+#include "uxMap.hpp"
+
+#endif // UX_HPP__

data/vendor/ux-trie/ux-0.1.9/src/uxMain.cpp

@@ -0,0 +1,206 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include "cmdline.h"
+#include "uxTrie.hpp"
+
+using namespace std;
+
+#include <time.h>
+#include <sys/time.h>
+#include <stdio.h>
+
+double gettimeofday_sec()
+{
+  struct timeval tv;
+  gettimeofday(&tv, NULL);
+  return tv.tv_sec + (double)tv.tv_usec*1e-6;
+}
+
+void analyzeKeyList(const vector<string>& keyList){
+  size_t lcs = 0;
+  for (size_t i = 1; i < keyList.size(); ++i){
+    const string& s1 = keyList[i-1];
+    const string& s2 = keyList[i];
+    size_t j = 0;
+    for (; j < s1.size() && j < s2.size() && s1[j] == s2[j]; ++j) {};
+    lcs += j;
+  }
+  cout << "  avelcs:\t" << (float)lcs / keyList.size() << endl;
+}
+
+
+size_t allKeySize(const vector<string>& keyList){
+  size_t ret = 0;
+  for (size_t i = 0; i < keyList.size(); ++i){
+    ret += keyList[i].size();
+  }
+  return ret;
+}
+
+void reportStat(const ux::Trie& ux, const vector<string>& keyList){
+  ux.allocStat(ux.getAllocSize(), cout);
+  ux.stat(cout);
+  analyzeKeyList(keyList);
+  size_t originalSize = allKeySize(keyList);
+  cout << "originalSize:\t" << allKeySize(keyList) << endl
+       << "   indexSize:\t" << ux.getAllocSize() << " (" << (float)ux.getAllocSize() / originalSize << ")" << endl
+       << "      keyNum:\t" << keyList.size() << endl;
+}
+
+void printQuery(const ux::Trie& ux,
+		const std::string& query,
+		const int limit){
+  cout << "query:[" << query << "]" << endl;
+  
+  // prefixSearch
+  size_t retLen = 0;
+  cout << "prefixSearch: ";
+  ux::id_t id = ux.prefixSearch(query.c_str(), query.size(), retLen);
+  if (id == ux::NOTFOUND){
+    cout << "not found." << endl;
+  } else {
+    cout << ux.decodeKey(id) << "\t(id=" << id << ")" << endl;
+  }
+
+  vector<ux::id_t> retIDs;  
+  // commonPrefixSearch
+  ux.commonPrefixSearch(query.c_str(), query.size(), retIDs, (size_t)limit);
+  cout << "commonPrefixSearch: " << retIDs.size() << " found." << endl;
+  for (size_t i = 0; i < retIDs.size(); ++i){
+    cout << ux.decodeKey(retIDs[i]) << "\t(id=" << retIDs[i] << ")" << endl;
+  }
+
+  // predictiveSearch
+  ux.predictiveSearch(query.c_str(), query.size(), retIDs, (size_t)limit);
+  cout << "predictiveSearch: " << retIDs.size() << " found." << endl;
+  for (size_t i = 0; i < retIDs.size(); ++i){
+    cout << ux.decodeKey(retIDs[i]) << "\t(id=" << retIDs[i] << ")" << endl;
+  }
+}
+
+
+int readKeyList(const string& fn, vector<string>& keyList){
+  ifstream ifs(fn.c_str());
+  if (!ifs){
+    cerr << "cannot open " << fn << endl;
+    return -1;
+  }
+
+  for (string key; getline(ifs, key); ){
+    if (key.size() > 0 &&
+	key[key.size()-1] == '\r'){
+      key = key.substr(0, key.size()-1);
+    }
+    keyList.push_back(key);
+  }
+  return 0;
+} 
+
+void performanceTest(ux::Trie& ux, vector<string>& keyList){
+  random_shuffle(keyList.begin(), keyList.end());
+  
+  double start = gettimeofday_sec();
+  size_t dummy = 0;
+  for (size_t i = 0; i < keyList.size() && i < 1000; ++i){
+    size_t retLen = 0;
+    dummy += ux.prefixSearch(keyList[i].c_str(), keyList[i].size(), retLen);
+  }
+  double end   = gettimeofday_sec();
+  cout << "  query time:\t" << end - start << endl; 
+  cout << "  check keys:\t" << min((int)keyList.size(), 1000) << endl;
+  
+  if (dummy == 777){
+    cerr << "luckey" << endl;
+  }
+}
+
+int buildUX(const string& fn, const string& index, const bool uncompress, const int verbose){
+  vector<string> keyList;
+  if (readKeyList(fn, keyList) == -1){
+    return -1;
+  }
+  ux::Trie ux;
+  double start = gettimeofday_sec();
+  ux.build(keyList, !uncompress);
+  double elapsedTime = gettimeofday_sec() - start;
+  if (verbose >= 1){
+    cout << "  index time:\t" << elapsedTime << endl;
+    reportStat(ux, keyList);
+  }
+  if (verbose >= 2){
+    performanceTest(ux, keyList);
+  }
+
+  if (index == "") return 0;
+  int err = ux.save(index.c_str());
+  if (err != ux::Trie::SUCCESS){
+    cerr << ux.what(err) << " " << index << endl;
+    return -1;
+  }
+  return 0;
+}
+
+int searchUX(const string& index, const int limit){
+  ux::Trie ux;
+  int err = ux.load(index.c_str());
+  if (err != ux::Trie::SUCCESS){ 
+    cerr << ux.what(err) << " " << index << endl;
+    return -1;
+  }
+  cout << "read:" << ux.size() << " keys" << endl;
+  
+  string query;
+  for (;;){
+    putchar('>');
+    getline(cin, query);
+    if (query.size() == 0){
+      break;
+    }
+    printQuery(ux, query, limit);
+  }
+
+  return 0;
+}
+
+int listUX(const string& index){
+  ux::Trie ux;
+  int err = ux.load(index.c_str());
+  if (err != ux::Trie::SUCCESS){
+    cerr << ux.what(err) << " " << index << endl;
+    return -1;
+  }
+  
+  for (size_t i = 0; i < ux.size(); ++i){
+    cout << ux.decodeKey(i) <<  endl;
+  }
+  return 0;
+}
+
+
+int main(int argc, char* argv[]){
+  cmdline::parser p;
+  p.add<string>("keylist",    'k', "key list", false);
+  p.add<string>("index",      'i', "index",     true);
+  p.add<int>   ("limit",      'l', "limit at search", false, 10);
+  p.add        ("uncompress", 'u', "tail is uncompressed");
+  p.add        ("enumerate",  'e', "enumerate all keywords");
+  p.add<int>   ("verbose",    'v', "verbose mode", 0);
+  p.add("help", 'h', "this message");
+  p.set_program_name("ux");
+
+  if (!p.parse(argc, argv) || p.exist("help")){
+    cerr << p.usage() << endl;
+    return -1;
+  }
+
+  if (p.exist("keylist")){
+    return buildUX(p.get<string>("keylist"), p.get<string>("index"), p.exist("uncompress"), p.get<int>("verbose"));
+  } else if (p.exist("enumerate")){
+    return listUX(p.get<string>("index"));
+  } else {
+    return searchUX(p.get<string>("index"), p.get<int>("limit"));
+  }
+
+  return 0; // NOT COME 
+}

data/vendor/ux-trie/ux-0.1.9/src/uxMap.hpp

@@ -0,0 +1,248 @@
+/* 
+ *  Copyright (c) 2010 Daisuke Okanohara
+  * 
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ * 
+ *   1. Redistributions of source code must retain the above Copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above Copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ *   3. Neither the name of the authors nor the names of its contributors
+ *      may be used to endorse or promote products derived from this
+ *      software without specific prior written permission.
+ */
+
+#ifndef UX_MAP_HPP__
+#define UX_MAP_HPP__
+
+#include <vector>
+#include <cassert>
+#include <iostream>
+#include <map>
+#include <string>
+#include "uxTrie.hpp"
+
+namespace ux{
+
+/**
+ * Succict Map using UX
+ */
+template <class V>
+class Map{
+public:
+  /**
+   * Constructor
+   */
+  Map() : size_(0){}
+
+  /**
+   * Destructor
+   */
+  ~Map() {}
+
+  /**
+   * Build a map without values
+   * @param keys keys to be associated
+   */
+  void build(std::vector<std::string>& keys){
+    trie_.build(keys);
+    vs_.resize(trie_.size());
+  }
+  
+  /**
+   * Build a map from std::map
+   * @param m A std::map as an input
+   */
+  void build(const std::map<std::string, V>& m){
+    std::vector<std::string> wordList;
+    for (typename std::map<std::string, V>::const_iterator it = 
+	   m.begin(); it != m.end(); ++it){
+      wordList.push_back(it->first);
+    }
+    trie_.build(wordList);
+    vs_.resize(wordList.size());
+    for (typename std::map<std::string, V>::const_iterator it = 
+	   m.begin(); it != m.end(); ++it){
+      const std::string key = it->first;
+      if (set(key.c_str(), key.size(), it->second) != 0){
+	return;
+      }
+    }
+  }
+
+  /**
+   * Build a map from the vector of the pair of a key and a value
+   * @param kvs A vector of the pair of a key and vlaue
+   */
+  void build(const std::vector< std::pair<std::string, V> >& kvs){
+    std::vector<std::string> wordList;
+    for (size_t i = 0; i < kvs.size(); ++i){
+      wordList.push_back(kvs[i].first);
+    }
+
+    trie_.build(wordList);
+    vs_.resize(wordList.size());
+
+    for (size_t i = 0; i < kvs.size(); ++i){
+      const std::string key = kvs[i].first;
+      assert(set(key.c_str(), key.size(), kvs[i].second) == 0);
+    }
+  }
+
+  /**
+   * Get a value for a given key
+   * @param str the key
+   * @param len the length of str
+   * @param v An associated value for a key
+   * @return 0 on success and -1 if not found
+   */
+  int get(const char* str, size_t len, V& v) const {
+    size_t retLen = 0;
+    id_t id = trie_.prefixSearch(str, len, retLen);
+    if (id == NOTFOUND){
+      return -1;
+    } 
+    v = vs_[id];
+    return 0;
+  }
+
+  /**
+   * Set a value for a given key
+   * @param str the key
+   * @param len the length of str
+   * @param v  A value to be associated for a key
+   * @return 0 on success and -1 if not found
+   */
+  int set(const char* str, size_t len, const V& v){
+    size_t retLen = 0;
+    id_t id = trie_.prefixSearch(str, len, retLen);
+    if (id == NOTFOUND){
+      return -1;
+    }
+    vs_[id] = v;
+    return 0;
+  }
+
+  /**
+   * Return the longest key that matches the prefix of the query in the dictionary
+   * @param str the query
+   * @param len the length of the query
+   * @param retLen The length of the matched key in the dictionary 
+   * @param v The associated value for the key
+   * @return 0 if found and -1 if not found
+   */
+  int prefixSearch(const char* str, size_t len, size_t& retLen, V& v) const {
+    id_t id = trie_.prefixSearch(str, len, retLen);
+    if (id == NOTFOUND){
+      return -1;
+    }
+    v = vs_[id];
+    return 0;
+  }
+
+  /** 
+   * Return the all associated values that match the prefix of the query in the dictionary
+   * @param str the query
+   * @param len the length of the query
+   * @param vs The returned values associated for the input key
+   * @param limit The maximum number of matched keys
+   * @return The number of matched keys
+   */
+  size_t commonPrefixSearch(const char* str, size_t len, std::vector<V>& vs, size_t limit = LIMIT_DEFAULT) const {
+    vs.clear();
+    std::vector<id_t> retIDs;
+    commonPrefixSearch(str, len, retIDs, limit);
+    vs.resize(retIDs.size());
+    for (size_t i = 0; i < retIDs.size(); ++i){
+      vs[i] = vs_[retIDs[i]];
+    }
+    return vs.size();
+  }
+
+  /** 
+   * Return the all keys whose their prefixes  match the query 
+   * @param str the query
+   * @param len the length of the query
+   * @param vs The associated values for the input key
+   * @param limit The maximum number of matched keys
+   * @return The number of matched keys
+   */
+  size_t predictiveSearch(const char* str, size_t len, std::vector<V>& vs, size_t limit = LIMIT_DEFAULT) const {
+    vs.clear();
+    std::vector<id_t> retIDs;
+    predictiveSearch(str, len, retIDs, limit);
+    vs.resize(retIDs.size());
+    for (size_t i = 0; i < retIDs.size(); ++i){
+      vs[i] = vs_[retIDs[i]];
+    }
+    return vs.size();
+  }
+
+  /**
+   * Return the key for the given ID
+   * @param id The ID of the key
+   * @param ret The key for the given ID or empty if such ID does not exist
+   */ 
+  void decodeKey(const size_t ind, std::string& ret) const {
+    trie_.decodeKey(ind, ret);
+  }
+
+  /**
+   * Save the map in ostream
+   * @param os The ostream as an output 
+   * @return 0 on success, -1 on failure
+   */
+  int save(std::ostream& os) const {
+    trie_.save(os);
+    size_t vsSize = vs_.size();
+    os.write((const char*)&vsSize, sizeof(vsSize));
+    os.write((const char*)&vs_[0], sizeof(vs_[0]) * vs_.size());
+    if (!os){
+      return -1;
+    } else {
+      return 0;
+    }
+  }
+
+  /**
+   * Load the map from istream
+   * @param is The istream as an input
+   * @return 0 on success, -1 on failure
+   */
+  int load(std::istream& is){
+    trie_.load(is);
+    size_t vsSize = 0;
+    is.read((char*)&vsSize, sizeof(vsSize));
+    vs_.resize(vsSize);
+    is.read((char*)&vs_[0], sizeof(vs_[0]) * vs_.size());
+    if (!is){
+      return -1;
+    } else {
+      return 0;
+    }
+  }
+
+  /**
+   * Get the number of keys 
+   * @return the number of keys
+   */
+  size_t size() const {
+    return trie_.size();
+  }
+
+private:
+  Trie trie_;
+  std::vector<V> vs_;
+  size_t size_;
+};
+
+
+}
+
+
+#endif // TRIE_MAP_HPP__