text_ux 0.0.1

data/vendor/ux-trie/ux-0.1.9/src/uxTrie.hpp

@@ -0,0 +1,220 @@
+/* 
+ *  Copyright (c) 2010 Daisuke Okanohara
+  * 
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ * 
+ *   1. Redistributions of source code must retain the above Copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above Copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ *   3. Neither the name of the authors nor the names of its contributors
+ *      may be used to endorse or promote products derived from this
+ *      software without specific prior written permission.
+ */
+
+#ifndef UX_TRIE_HPP__
+#define UX_TRIE_HPP__
+
+#include <string>
+#include <vector>
+#include <stdint.h>
+#include "bitVec.hpp"
+#include "rsDic.hpp"
+
+namespace ux{
+
+typedef uint64_t id_t;
+
+enum {
+  NOTFOUND      = 0xFFFFFFFFU,
+  LIMIT_DEFAULT = 0xFFFFFFFFU
+};
+
+
+/**
+ * Succinct Trie Data structure
+ */
+class Trie {
+public:
+  /**
+   * Constructor
+   */
+  Trie();
+
+  /**
+   * Constructor 
+   * @param keyList input key list
+   * @param isTailUX use tail compression. 
+   */
+  Trie(std::vector<std::string>& keyList, bool isTailUX = true);
+
+  /**
+   * Destructor
+   */
+  ~Trie();
+
+  /**
+   * Build a dictionary from keyList
+   * @param keyList input key list
+   * @param isTailUX use tail compression. 
+   */
+  void build(std::vector<std::string>& keyList, bool isTailUX = true);
+  
+  /**
+   * Save the dictionary in a file
+   * @param indexName The file name
+   * @return 0 on success, -1 on failure
+   */
+  int save(const char* indexName) const;
+
+  /**
+   * Load the dicitonary from a file
+   * @param indexName The filename
+   * @return 0 on success, -1 on failure
+   */
+  int load(const char* indexName);
+
+  /**
+   * Save the dictionary in ostream
+   * @param os The ostream as an output 
+   * @return 0 on success, -1 on failure
+   */
+  int save(std::ostream& os) const;
+
+  /**
+   * Load the dictionary from istream
+   * @param is The istream as an input
+   * @return 0 on success, -1 on failure
+   */
+  int load(std::istream& is);
+
+  /**
+   * Return the longest key that matches the prefix of the query in the dictionary
+   * @param str the query
+   * @param len the length of the query
+   * @param retLen The length of the matched key in the dictionary 
+   * @return The ID of the matched key or NOTFOUND if no key is matched
+   */
+  id_t prefixSearch(const char* str, size_t len, size_t& retLen) const;
+
+  /** 
+   * Return the all keys that match the prefix of the query in the dictionary
+   * @param str the query
+   * @param len the length of the query
+   * @param retIDs The IDs of the matched keys
+   * @param limit The maximum number of matched keys
+   * @return The number of matched keys
+   */
+  size_t commonPrefixSearch(const char* str, size_t len, std::vector<id_t>& retIDs, 
+			    size_t limit = LIMIT_DEFAULT) const;
+
+  /** 
+   * Return the all keys whose their prefixes  match the query 
+   * @param str the query
+   * @param len the length of the query
+   * @param The IDs of the matched keys
+   * @param limit The maximum number of matched keys
+   * @return The number of matched keys
+   */
+  size_t predictiveSearch(const char* str, size_t len, std::vector<id_t>& retIDs, 
+			  size_t limit = LIMIT_DEFAULT) const;
+  
+  /**
+   * Return the key for the given ID
+   * @param id The ID of the key
+   * @param ret The key for the given ID or empty if such ID does not exist
+   */ 
+  void decodeKey(id_t id, std::string& ret) const;
+
+  /**
+   * Return the key for the given ID
+   * @param id The ID of the key
+   * @return The key for the given ID or empty if such ID does not exist
+   */ 
+  std::string decodeKey(id_t id) const;
+  
+  /**
+   * Return the number of keys in the dictionary
+   * @return the number of keys in the dictionary
+   */
+  size_t size() const;
+
+  /**
+   * Clear the internal state
+   */
+  void clear();
+  
+  /*
+   * Get the allocated memory size
+   * @return The size of the allocated memory
+   */
+  size_t getAllocSize() const;
+  
+  /*
+   * Report the statistics of the memory allocation
+   * @param allocSize The initial overhead size
+   * @param os The output distination
+   */
+  void allocStat(size_t allocSize, std::ostream& os) const;
+
+  /*
+   * Report the internal statistics 
+   * @param os The output distination
+   */
+  void stat(std::ostream& os) const;
+
+  /**
+   * Report the error message for the error ID
+   * @param error The error ID
+   * @return The error message
+   */
+  static std::string what(int error);
+
+private:
+  void buildTailUX();
+  bool isLeaf(uint64_t pos) const;
+  void getChild(uint8_t c, uint64_t& pos, uint64_t& zeros) const;
+  void getParent(uint8_t& c, uint64_t& pos, uint64_t& zeros) const;
+  void traverse(const char* str, size_t len, size_t& retLen, std::vector<id_t>& retIDs, 
+		size_t limit) const;
+
+  void enumerateAll(uint64_t pos, uint64_t zeros, std::vector<id_t>& retIDs, size_t limit) const;
+  bool tailMatch(const char* str, size_t len, size_t depth,
+		 uint64_t tailID, size_t& retLen) const;
+  std::string getTail(uint64_t i) const;
+
+  RSDic loud_;
+  RSDic terminal_;
+  RSDic tail_;
+
+  std::vector<std::string> vtails_;
+  Trie* vtailux_;
+  std::vector<uint8_t> edges_;
+  BitVec tailIDs_;
+  size_t tailIDLen_;
+  size_t keyNum_;
+  bool isReady_;
+
+public:
+  /** 
+   * Error code.
+   */
+  enum {
+    SUCCESS          = 0, 
+    FILE_OPEN_ERROR  = 1,
+    FILE_WRITE_ERROR = 2,
+    FILE_READ_ERROR  = 3,
+    SAVE_ERROR       = 4,
+    LOAD_ERROR       = 5
+  };
+};
+
+
+}
+
+#endif // UX_TRIE_HPP__

data/vendor/ux-trie/ux-0.1.9/src/uxUtil.cpp

@@ -0,0 +1,92 @@
+/* 
+ *  Copyright (c) 2010 Daisuke Okanohara
+ * 
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ * 
+ *   1. Redistributions of source code must retain the above Copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above Copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ *   3. Neither the name of the authors nor the names of its contributors
+ *      may be used to endorse or promote products derived from this
+ *      software without specific prior written permission.
+ */
+
+#include "uxUtil.hpp"
+
+namespace ux{
+
+uint64_t lg2(const uint64_t x){
+  uint64_t ret = 0;
+  while (x >> ret){
+    ++ret;
+  }
+  return ret;
+}
+  
+uint64_t mask(uint64_t x, uint64_t pos){
+  return x & ((1LLU << pos) - 1);
+}
+
+uint64_t popCount(uint64_t r) {
+  r = (r & 0x5555555555555555ULL) +
+    ((r >> 1) & 0x5555555555555555ULL);
+  r = (r & 0x3333333333333333ULL) +
+    ((r >> 2) & 0x3333333333333333ULL);
+  r = (r + (r >> 4)) & 0x0f0f0f0f0f0f0f0fULL;
+  r = r + (r >>  8);
+  r = r + (r >> 16);
+  r = r + (r >> 32);
+  return (uint64_t)(r & 0x7f);
+}
+
+uint64_t popCountMasked(uint64_t x, uint64_t pos){
+  return popCount(mask(x, pos));
+}
+
+uint64_t selectBlock(uint64_t r, uint64_t x, uint8_t b) {
+  if (!b) x = ~x;
+  uint64_t x1 = x - ((x & 0xAAAAAAAAAAAAAAAALLU) >> 1);
+  uint64_t x2 = (x1 & 0x3333333333333333LLU) + ((x1 >> 2) & 0x3333333333333333LLU);
+  uint64_t x3 = (x2 + (x2 >> 4)) & 0x0F0F0F0F0F0F0F0FLLU;
+  
+  uint64_t pos = 0;
+  for (;;  pos += 8){
+    uint64_t b = (x3 >> pos) & 0xFFLLU;
+    if (r <= b) break;
+    r -= b;
+  }
+
+  uint64_t v2 = (x2 >> pos) & 0xFLLU;
+  if (r > v2) {
+    r -= v2;
+    pos += 4;
+  }
+
+  uint64_t v1 = (x1 >> pos) & 0x3LLU;
+  if (r > v1){
+    r -= v1;
+    pos += 2;
+  }
+
+  uint64_t v0  = (x >> pos) & 0x1LLU;
+  if (v0 < r){
+    r -= v0;
+    pos += 1;
+  }
+
+  return pos;
+}
+
+uint64_t getBitNum(uint64_t oneNum, uint64_t num, uint8_t bit){
+   if (bit) return oneNum;
+   else     return num - oneNum;
+}
+
+}
+

data/vendor/ux-trie/ux-0.1.9/src/uxUtil.hpp

@@ -0,0 +1,35 @@
+/* 
+ *  Copyright (c) 2010 Daisuke Okanohara
+ * 
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ * 
+ *   1. Redistributions of source code must retain the above Copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above Copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ *   3. Neither the name of the authors nor the names of its contributors
+ *      may be used to endorse or promote products derived from this
+ *      software without specific prior written permission.
+ */
+
+#ifndef UX_UTIL_HPP__
+#define UX_UTIL_HPP__
+
+#include <stdint.h>
+
+namespace ux {
+  uint64_t lg2(uint64_t x);
+  uint64_t mask(uint64_t x, uint64_t pos);
+  uint64_t popCount(uint64_t r);
+  uint64_t popCountMasked(uint64_t x, uint64_t pos);
+  uint64_t selectBlock(uint64_t pos, uint64_t x, uint8_t b);
+  uint64_t getBitNum(uint64_t oneNum, uint64_t num, uint8_t bit);
+}
+
+#endif // UX_UTIL_HPP__
+

data/vendor/ux-trie/ux-0.1.9/src/wscript

@@ -0,0 +1,43 @@
+APPNAME= 'ux'
+
+top = '.'
+out = 'bin'
+
+def options(ctx):
+  pass
+    
+def configure(ctx):
+  pass
+
+def build(bld):
+  bld.shlib(
+       source       = 'uxTrie.cpp bitVec.cpp rsDic.cpp uxUtil.cpp uxMap.cpp',
+       target       = 'ux',
+       name         = 'UX',
+       includes     = '.')
+  bld.program(
+       source       = 'uxMain.cpp',
+       target       = 'ux',
+       includes     = '.',
+       use          = 'UX')
+  bld.program(
+       features     = 'gtest',
+       source       = 'uxTest.cpp',
+       target       = 'ux_test',
+       use          = 'UX' ,
+       includes     = '.')
+  bld.program(
+       features     = 'gtest',
+       source       = 'bitVecTest.cpp',
+       target       = 'bitvec_test',
+       use          = 'UX',
+       includes     = '.')
+
+  bld.program(
+       features     = 'gtest',
+       source       = 'uxMapTest.cpp',
+       target       = 'uxmap_test',
+       use          = 'UX',
+       includes     = '.')
+
+  bld.install_files('${PREFIX}/include/ux', bld.path.ant_glob('*.hpp'))

data/vendor/ux-trie/ux-0.1.9/unittestt.py

@@ -0,0 +1,166 @@
+#! /usr/bin/env python
+# encoding: utf-8
+
+import os, sys
+import Options, Task, Utils, Logs
+from TaskGen import before, after, feature
+from Constants import *
+
+def detect(conf):
+    if conf.check_cfg(path = 'gtest-config',
+                      args = '--cppflags --cxxflags --ldflags --libs',
+                      package = '',
+                      uselib_store = 'GTEST'):
+        def f(str):
+            if str == 'gtest':
+                return 'gtest_main'
+            else:
+                return str
+        conf.env.LIB_GTEST = map(f, conf.env.LIB_GTEST)
+
+def set_options(opt):
+    opt.add_option('--check', action = 'store_true', default = False,
+                   help = 'Execute unit tests')
+    opt.add_option('--checkall', action = 'store_true', default = False,
+                   help = 'Execute all unit tests')
+    opt.add_option('--checkone', action = 'store', default = False,
+                   help = 'Execute specified unit test')
+    opt.add_option('--checkfilter', action = 'store', default = False,
+                   help = 'Execute unit tests sprcified by pattern')
+
+def match_filter(filt, targ):
+    if isinstance(filt, str):
+        (pat, _, _) = filt.partition('.')
+        if pat == '*':
+            return True
+        return pat == targ
+    return False
+
+def test_remover(self):
+    if not Options.options.check and not Options.options.checkall and self.target != Options.options.checkone and not match_filter(Options.options.checkfilter, self.target):
+        self.meths[:] = []
+
+feature('testt', 'gtest')(test_remover)
+before('apply_core')(test_remover)
+
+def make_test(self):
+    if not 'cprogram' in self.features:
+        Logs.error('test cannot be executed %s'%self)
+        return
+    self.default_install_path = None
+    self.create_task('utest', self.link_task.outputs)
+
+feature('testt', 'gtest')(make_test)
+after('apply_link', 'vars_target_cprogram')(make_test)
+
+def gtest_attach(self):
+    if not self.env.HAVE_GTEST:
+        Logs.error('gtest is not found')
+        self.meths[:] = []
+        return
+
+    if isinstance(self.uselib, str):
+        self.uselib += " GTEST"
+    else:
+        self.uselib.append('GTEST')
+
+feature('gtest')(gtest_attach)
+before('apply_core')(gtest_attach)
+
+import threading
+testlock = threading.Lock()
+
+def exec_test(self):
+    variant = self.env.variant()
+    filename = self.inputs[0].abspath(self.env)
+
+    try:
+        fu = getattr(self.generator.bld, 'all_test_paths')
+    except AttributeError:
+        fu = os.environ.copy()
+        self.generator.bld.all_test_paths = fu
+
+        lst = []
+        for obj in self.generator.bld.all_task_gen:
+            link_task=getattr(obj, 'link_task', None)
+            if link_task and link_task.env.variant()==variant:
+                lst.append(link_task.outputs[0].parent.abspath(obj.env))
+
+        def add_path(dct, path, var):
+            dct[var] = os.pathsep.join(Utils.to_list(path)+[os.environ.get(var, '')])
+        if sys.platform=='win32':
+            add_path(fu, lst, 'PATH')
+        elif sys.platform=='darwin':
+            add_path(fu, lst, 'DYLD_LIBRARY_PATH')
+            add_path(fu, lst, 'LD_LIBRARY_PATH')
+        else:
+            add_path(fu, lst, 'LD_LIBRARY_PATH')
+
+    cmdline = [filename]
+    cwd = self.inputs[0].parent.abspath(self.env)
+
+    if isinstance(Options.options.checkfilter, str):
+        (_, _, filt) = Options.options.checkfilter.partition('.')
+        if filt != "":
+            cmdline += ['--gtest_filter=' + filt]
+
+    proc = Utils.pproc.Popen(cmdline, cwd = cwd, env = fu,
+                             stderr = Utils.pproc.PIPE,
+                             stdout = Utils.pproc.PIPE)
+    (stdout, stderr) = proc.communicate()
+    tup = (filename, proc.returncode, stdout, stderr)
+
+    testlock.acquire()
+    try:
+        try:
+            self.generator.bld.utest_results.append(tup)
+        except:
+            self.generator.bld.utest_results = [tup]
+
+        a = getattr(self.generator.bld, 'added_post_fun', False)
+        if not a:
+            self.generator.bld.add_post_fun(summary)
+            self.generator.bld.added_post_fun = True
+
+    finally:
+        testlock.release()
+
+cls = Task.task_type_from_func('utest', func = exec_test, color = 'PINK', ext_in = '.bin')
+old = cls.runnable_status
+
+def test_status(self):
+    if Options.options.checkall:
+        return RUN_ME
+    if Options.options.checkone == self.generator.name:
+        return RUN_ME
+    if isinstance(Options.options.checkfilter, str):
+        if match_filter(Options.options.checkfilter, self.generator.name):
+            return RUN_ME
+    return old(self)
+
+cls.runnable_status = test_status
+cls.quiet = 1
+
+def summary(bld):
+    lst = getattr(bld, 'utest_results', [])
+
+    if not lst: return
+
+    total = len(lst)
+    fail = len([x for x in lst if x[1]])
+
+    Utils.pprint('CYAN', 'test summary')
+    Utils.pprint('CYAN', '  tests that pass %d/%d' % (total-fail, total))
+
+    for (f, code, out, err) in lst:
+        if not code:
+            Utils.pprint('GREEN', '    %s' % f)
+            if isinstance(Options.options.checkfilter, str):
+                print(out)
+
+    if fail>0:
+        Utils.pprint('RED', '  tests that fail %d/%d' % (fail, total))
+        for (f, code, out, err) in lst:
+            if code:
+                Utils.pprint('RED', '    %s' % f)
+                print(out)