jieba_rb 0.0.1

data/ext/cppjieba/src/Limonp/BlockingQueue.hpp

@@ -0,0 +1,128 @@
+/*
+https://github.com/chenshuo/muduo/blob/master/muduo/base/BlockingQueue.h
+*/
+
+#ifndef LIMONP_BLOCKINGQUEUE_HPP
+#define LIMONP_BLOCKINGQUEUE_HPP
+
+#include <queue>
+#include "BoundedQueue.hpp"
+#include "Condition.hpp"
+
+namespace Limonp
+{
+    template<class T>
+        class BlockingQueue: NonCopyable
+        {
+            public:
+                BlockingQueue()
+                    : mutex_(), notEmpty_(mutex_), queue_()
+                {
+                }
+
+                void push(const T& x)
+                {
+                    MutexLockGuard lock(mutex_);
+                    queue_.push(x);
+                    notEmpty_.notify(); // wait morphing saves us
+                }
+
+                T pop()
+                {
+                    MutexLockGuard lock(mutex_);
+                    // always use a while-loop, due to spurious wakeup
+                    while (queue_.empty())
+                    {
+                        notEmpty_.wait();
+                    }
+                    assert(!queue_.empty());
+                    T front(queue_.front());
+                    queue_.pop();
+                    return front;
+                }
+
+                size_t size() const
+                {
+                    MutexLockGuard lock(mutex_);
+                    return queue_.size();
+                }
+                bool empty() const
+                {
+                    return size() == 0;
+                }
+
+            private:
+                mutable MutexLock mutex_;
+                Condition         notEmpty_;
+                std::queue<T>     queue_;
+        };
+
+    template<typename T>
+        class BoundedBlockingQueue : NonCopyable
+        {
+            public:
+                explicit BoundedBlockingQueue(size_t maxSize)
+                    : mutex_(),
+                    notEmpty_(mutex_),
+                    notFull_(mutex_),
+                    queue_(maxSize)
+                {}
+
+                void push(const T& x)
+                {
+                    MutexLockGuard lock(mutex_);
+                    while (queue_.full())
+                    {
+                        notFull_.wait();
+                    }
+                    assert(!queue_.full());
+                    queue_.push(x);
+                    notEmpty_.notify();
+                }
+
+                T pop()
+                {
+                    MutexLockGuard lock(mutex_);
+                    while (queue_.empty())
+                    {
+                        notEmpty_.wait();
+                    }
+                    assert(!queue_.empty());
+                    T res = queue_.pop();
+                    notFull_.notify();
+                    return res;
+                }
+
+                bool empty() const
+                {
+                    MutexLockGuard lock(mutex_);
+                    return queue_.empty();
+                }
+
+                bool full() const
+                {
+                    MutexLockGuard lock(mutex_);
+                    return queue_.full();
+                }
+
+                size_t size() const
+                {
+                    MutexLockGuard lock(mutex_);
+                    return queue_.size();
+                }
+
+                size_t capacity() const
+                {
+                    return queue_.capacity();
+                }
+
+            private:
+                mutable MutexLock          mutex_;
+                Condition                  notEmpty_;
+                Condition                  notFull_;
+                BoundedQueue<T>  queue_;
+        };
+
+}
+
+#endif

data/ext/cppjieba/src/Limonp/BoundedQueue.hpp

@@ -0,0 +1,73 @@
+#ifndef LIMONP_BOUNDED_QUEUE_HPP
+#define LIMONP_BOUNDED_QUEUE_HPP
+
+#include <vector>
+#include <fstream>
+#include <cassert>
+
+namespace Limonp
+{
+    using namespace std;
+    template<class T>
+        class BoundedQueue
+        {
+            private:
+                size_t head_;
+                size_t tail_;
+                size_t size_;
+                const size_t capacity_;
+                vector<T> circular__buffer;
+            public:
+                explicit BoundedQueue(size_t capacity): capacity_(capacity), circular__buffer(capacity)
+                {
+                    head_ = 0;
+                    tail_ = 0;
+                    size_ = 0;
+                    assert(capacity_);
+                }
+                ~BoundedQueue(){}
+            public:
+                void clear()
+                {
+                    head_ = 0;
+                    tail_ = 0;
+                    size_ = 0;
+                }
+                bool empty() const
+                {
+                    return !size_;
+                }
+                bool full() const
+                {
+                    return capacity_ == size_;
+                }
+                size_t size() const
+                {
+                    return size_;
+                }
+                size_t capacity() const
+                {
+                    return capacity_;
+                }
+
+                void push(const T& t)
+                {
+                    assert(!full());
+                    circular__buffer[tail_] = t;
+                    tail_ = (tail_ + 1) % capacity_;
+                    size_ ++;
+                }
+
+                T pop()
+                {
+                    assert(!empty());
+                    size_t oldPos = head_;
+                    head_ = (head_ + 1) % capacity_;
+                    size_ --;
+                    return circular__buffer[oldPos];
+                }
+
+        };
+}
+
+#endif

data/ext/cppjieba/src/Limonp/CastFloat.hpp

@@ -0,0 +1,90 @@
+#ifndef LIMONP_CAST_FUNCTS_H
+#define LIMONP_CAST_FUNCTS_H
+
+namespace Limonp
+{
+    namespace CastFloat
+    {
+        //logical and or
+        static const int sign_32 = 0xC0000000;
+        static const int exponent_32 = 0x07800000;
+        static const int mantissa_32 = 0x007FE000;
+        static const int sign_exponent_32 = 0x40000000;
+        static const int loss_32 = 0x38000000;
+
+        static const short sign_16 = (short)0xC000;
+        static const short exponent_16 = (short)0x3C00;
+        static const short mantissa_16 = (short)0x03FF;
+        static const short sign_exponent_16 = (short)0x4000;
+        static const int exponent_fill_32 = 0x38000000;
+
+        //infinite
+        static const short infinite_16 = (short) 0x7FFF;
+        static const short infinitesmall_16 = (short) 0x0000;
+
+        inline float intBitsToFloat(unsigned int x)
+        {
+            union
+            {
+                float f;
+                int i;
+            }u;
+            u.i = x;
+            return u.f;
+        }
+
+        inline int floatToIntBits(float f)
+        {
+            union
+            {
+                float f;
+                int i ;
+            }u;
+            u.f = f;
+            return u.i;
+        }
+
+        inline short floatToShortBits(float f)
+        {
+            int fi = floatToIntBits(f);
+
+            // 提取关键信息
+            short sign = (short) ((unsigned int)(fi & sign_32) >> 16);
+            short exponent = (short) ((unsigned int)(fi & exponent_32) >> 13);
+            short mantissa = (short) ((unsigned int)(fi & mantissa_32) >> 13);
+            // 生成编码结果
+            short code = (short) (sign | exponent | mantissa);
+            // 无穷大量、无穷小量的处理
+            if ((fi & loss_32) > 0 && (fi & sign_exponent_32) > 0) {
+                // 当指数符号为1时(正次方)，且左234位为1，返回无穷大量
+                return (short) (code | infinite_16);
+            }
+            if (((fi & loss_32) ^ loss_32) > 0 && (fi & sign_exponent_32) == 0) {
+                // 当指数符号位0时(负次方)，且左234位为0(与111异或>0)，返回无穷小量
+                return infinitesmall_16;
+            }
+
+            return code;
+        }
+
+        inline float shortBitsToFloat(short s)
+        {
+            /*
+             * 指数空余3位：若符号位为1，补0；若符号位为0，补1。 尾数位在后补0(13个)
+             */
+            int sign = ((int) (s & sign_16)) << 16;
+            int exponent = ((int) (s & exponent_16)) << 13;
+            // 指数符号位为0，234位补1
+            if ((s & sign_exponent_16) == 0 && s != 0) {
+                exponent |= exponent_fill_32;
+            }
+            int mantissa = ((int) (s & mantissa_16)) << 13;
+            // 生成解码结果
+            int code = sign | exponent | mantissa;
+            return intBitsToFloat(code);
+
+        }
+    }
+}
+
+#endif

data/ext/cppjieba/src/Limonp/Condition.hpp

@@ -0,0 +1,48 @@
+/*
+ * https://github.com/chenshuo/muduo/blob/master/muduo/base/Condition.h
+ */
+
+#ifndef LIMONP_CONDITION_HPP
+#define LIMONP_CONDITION_HPP
+
+#include "MutexLock.hpp"
+
+namespace Limonp
+{
+    class Condition : NonCopyable
+    {
+        public:
+            explicit Condition(MutexLock& mutex)
+                : mutex_(mutex)
+            {
+                LIMONP_CHECK(!pthread_cond_init(&pcond_, NULL));
+            }
+
+            ~Condition()
+            {
+                LIMONP_CHECK(!pthread_cond_destroy(&pcond_));
+            }
+
+            void wait()
+            {
+                LIMONP_CHECK(!pthread_cond_wait(&pcond_, mutex_.getPthreadMutex()));
+            }
+
+            void notify()
+            {
+                LIMONP_CHECK(!pthread_cond_signal(&pcond_));
+            }
+
+            void notifyAll()
+            {
+                LIMONP_CHECK(!pthread_cond_broadcast(&pcond_));
+            }
+
+        private:
+            MutexLock& mutex_;
+            pthread_cond_t pcond_;
+    };
+
+}
+
+#endif

data/ext/cppjieba/src/Limonp/Config.hpp

@@ -0,0 +1,118 @@
+/************************************
+ * file enc : utf8
+ * author   : wuyanyi09@gmail.com
+ ************************************/
+#ifndef LIMONP_CONFIG_H
+#define LIMONP_CONFIG_H
+
+
+#include <map>
+#include <fstream>
+#include <iostream>
+#include <assert.h>
+#include "StringUtil.hpp"
+
+namespace Limonp
+{
+    using namespace std;
+    class Config
+    {
+        public:
+            explicit Config(const string& filePath)
+            {
+                loadFile_(filePath);
+            }
+        public:
+            operator bool ()
+            {
+                return !map_.empty();
+            }
+        private:
+            void loadFile_(const string& filePath)
+            {
+                ifstream ifs(filePath.c_str());
+                assert(ifs);
+                string line;
+                vector<string> vecBuf;
+                size_t lineno = 0;
+                while(getline(ifs, line))
+                {
+                    lineno ++;
+                    trim(line);
+                    if(line.empty() || startsWith(line, "#"))
+                    {
+                        continue;
+                    }
+                    vecBuf.clear();
+                    if(!split(line, vecBuf, "=") || 2 != vecBuf.size())
+                    {
+                        fprintf(stderr, "line[%s] illegal.\n", line.c_str());
+                        assert(false);
+                        continue;
+                    }
+                    string& key = vecBuf[0];
+                    string& value = vecBuf[1];
+                    trim(key);
+                    trim(value);
+                    if(!map_.insert(make_pair(key, value)).second)
+                    {
+                        fprintf(stderr, "key[%s] already exits.\n", key.c_str());
+                        assert(false);
+                        continue;
+                    }
+                }
+                ifs.close();
+            }
+        public:
+            bool get(const string& key, string& value) const
+            {
+                map<string, string>::const_iterator it = map_.find(key);
+                if(map_.end() != it)
+                {
+                    value = it->second;
+                    return true;
+                }
+                return false;
+            }
+            bool get(const string& key, int & value) const
+            {
+                string str;
+                if(!get(key, str)) {
+                    return false;
+                }
+                value = atoi(str.c_str());
+                return true;
+            }
+            const char* operator [] (const char* key) const
+            {
+                if(NULL == key)
+                {
+                    return NULL;
+                }
+                map<string, string>::const_iterator it = map_.find(key);
+                if(map_.end() != it)
+                {
+                    return it->second.c_str();
+                }
+                return NULL;
+            }
+        public:
+            string getConfigInfo() const
+            {
+                string res;
+                res << *this;
+                return res;
+            }
+        private:
+            map<string, string> map_;
+        private:
+            friend ostream& operator << (ostream& os, const Config& config);
+    };
+    
+    inline ostream& operator << (ostream& os, const Config& config)
+    {
+        return os << config.map_;
+    }
+}
+
+#endif

data/ext/cppjieba/src/Limonp/HandyMacro.hpp

@@ -0,0 +1,31 @@
+#ifndef LIMONP_HANDY_MACRO_HPP
+#define LIMONP_HANDY_MACRO_HPP
+
+#include <cstdio>
+#include <cstdlib>
+
+#define LIMONP_CHECK(exp) \
+    if(!(exp)){fprintf(stderr, "File:%s, Line:%d Exp:[" #exp "] is true, abort.\n", __FILE__, __LINE__); abort();}
+
+#define print(x) cout<< #x": " << x <<endl
+/*
+#define XX_GET_SET(varType, varName, funName)\
+private: varType varName;\
+public: inline varType get##funName(void) const {return varName;}\
+public: inline void set##funName(varType var) {varName = var;}
+
+#define XX_GET(varType, varName, funName)\
+private: varType varName;\
+public: inline varType get##funName(void) const {return varName;}
+
+#define XX_SET(varType, varName, funName)\
+private: varType varName;\
+public: inline void set##funName(varType var) {varName = var;}
+
+#define XX_GET_SET_BY_REF(varType, varName, funName)\
+private: varType varName;\
+public: inline const varType& get##funName(void) const {return varName;}\
+public: inline void set##funName(const varType& var){varName = var;}
+*/
+
+#endif

data/ext/cppjieba/src/Limonp/InitOnOff.hpp

@@ -0,0 +1,21 @@
+#ifndef LIMONP_INITONOFF_H
+#define LIMONP_INITONOFF_H
+
+namespace Limonp
+{
+    class InitOnOff
+    {
+        public:
+            InitOnOff():isInited_(false){};
+            ~InitOnOff(){};
+        protected:
+            bool isInited_;
+            bool getInitFlag_()const{return isInited_;};
+            bool setInitFlag_(bool flag){return isInited_ = flag;};
+        public:
+            operator bool() const {return getInitFlag_();};
+
+    };
+}
+
+#endif

data/ext/cppjieba/src/Limonp/LocalVector.hpp

@@ -0,0 +1,171 @@
+#ifndef LIMONP_LOCAL_VECTOR_HPP
+#define LIMONP_LOCAL_VECTOR_HPP
+
+#include <iostream>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+
+namespace Limonp
+{
+    using namespace std;
+    /*
+     * LocalVector<T> : T must be primitive type (char , int, size_t), if T is struct or class, LocalVector<T> may be dangerous..
+     * LocalVector<T> is simple and not well-tested. 
+     */
+    const size_t LOCAL_VECTOR_BUFFER_SIZE = 16;
+    template <class T>
+        class LocalVector
+        {
+            public:
+                typedef const T* const_iterator ;
+                typedef T value_type;
+                typedef size_t size_type;
+            private:
+                T buffer_[LOCAL_VECTOR_BUFFER_SIZE];
+                T * ptr_;
+                size_t size_;
+                size_t capacity_;
+            public:
+                LocalVector()
+                {
+                    init_();
+                };
+                LocalVector(const LocalVector<T>& vec)
+                {
+                    init_();
+                    *this = vec;
+                }
+                LocalVector(const_iterator  begin, const_iterator end) // TODO: make it faster
+                {
+                    init_();
+                    while(begin != end)
+                    {
+                        push_back(*begin++);
+                    }
+                }
+                LocalVector(size_t size, const T& t) // TODO: make it faster
+                {
+                    init_();
+                    while(size--)
+                    {
+                        push_back(t);
+                    }
+                }
+                ~LocalVector()
+                {
+                    if(ptr_ != buffer_)
+                    {
+                        free(ptr_);
+                    }
+                };
+            public:
+                LocalVector<T>& operator = (const LocalVector<T>& vec)
+                {
+                    clear();
+                    size_ = vec.size();
+                    capacity_ = vec.capacity();
+                    if(vec.buffer_ == vec.ptr_)
+                    {
+                        memcpy(buffer_, vec.buffer_, sizeof(T) * size_);
+                        ptr_ = buffer_;
+                    }
+                    else
+                    {
+                        ptr_ = (T*) malloc(vec.capacity() * sizeof(T));
+                        assert(ptr_);
+                        memcpy(ptr_, vec.ptr_, vec.size() * sizeof(T));
+                    }
+                    return *this;
+                }
+            private:
+                void init_()
+                {
+                    ptr_ = buffer_;
+                    size_ = 0;
+                    capacity_ = LOCAL_VECTOR_BUFFER_SIZE;
+                }
+            public:
+                T& operator [] (size_t i) 
+                {
+                    return ptr_[i];
+                }
+                const T& operator [] (size_t i) const
+                {
+                    return ptr_[i];
+                }
+                void push_back(const T& t)
+                {
+                    if(size_ == capacity_)
+                    {
+                        assert(capacity_);
+                        reserve(capacity_ * 2);
+                    }
+                    ptr_[size_ ++ ] = t;
+                }
+                void reserve(size_t size) 
+                {
+                    if(size <= capacity_)
+                    {
+                        return;
+                    }
+                    T * next =  (T*)malloc(sizeof(T) * size);
+                    assert(next);
+                    T * old = ptr_;
+                    ptr_ = next;
+                    memcpy(ptr_, old, sizeof(T) * capacity_);
+                    capacity_ = size;
+                    if(old != buffer_)
+                    {
+                        free(old);
+                    }
+                }
+                bool empty() const
+                {
+                    return 0 == size();
+                }
+                size_t size() const
+                {
+                    return size_;
+                }
+                size_t capacity() const
+                {
+                    return capacity_;
+                }
+                const_iterator begin() const
+                {
+                    return ptr_;
+                }
+                const_iterator end() const
+                {
+                    return ptr_ + size_;
+                }
+                void clear()
+                {
+                    if(ptr_ != buffer_)
+                    {
+                        free(ptr_);
+                    }
+                    init_();
+                }
+        };
+
+    template <class T>
+        ostream & operator << (ostream& os, const LocalVector<T>& vec)
+        {
+            if(vec.empty())
+            {
+                return os << "[]";
+            }
+            os<<"[\""<<vec[0];
+            for(size_t i = 1; i < vec.size(); i++)
+            {
+                os<<"\", \""<<vec[i];
+            }
+            os<<"\"]";
+            return os;
+        }
+
+}
+
+#endif