ooxml_crypt 0.1.0

data/vendor/cybozulib/include/cybozu/mutex.hpp

@@ -0,0 +1,144 @@
+#pragma once
+/**
+	@file
+	@brief mutex
+
+	@author MITSUNARI Shigeo(@herumi)
+	@author MITSUNARI Shigeo
+*/
+
+#ifdef _WIN32
+	#ifndef WIN32_LEAN_AND_MEAN
+		#define WIN32_LEAN_AND_MEAN
+	#endif
+	#include <windows.h>
+#else
+	#include <pthread.h>
+	#include <time.h>
+#endif
+#include <assert.h>
+#include <stdlib.h>
+
+namespace cybozu {
+
+class ConditionVariable;
+
+namespace thread {
+
+#ifdef _WIN32
+	typedef HANDLE MutexHandle;
+	inline void MutexInit(MutexHandle& mutex)
+	{
+//		mutex = CreateSemaphore(NULL /* no security */, 1 /* init */, 0x7FFFFFFF /* max */, NULL /* no name */);
+		mutex = CreateMutex(NULL /* no security */, FALSE /* no owner */, NULL /* no name */);
+	}
+	inline void MutexLock(MutexHandle& mutex) { WaitForSingleObject(mutex, INFINITE); }
+	/*
+		return false if timeout
+		@param msec [in] msec
+	*/
+	inline bool MutexLockTimeout(MutexHandle& mutex, int msec)
+	{
+		DWORD ret = WaitForSingleObject(mutex, msec);
+		if (ret == WAIT_OBJECT_0) {
+			return true;
+		}
+		if (ret == WAIT_TIMEOUT) {
+			return false;
+		}
+		/* ret == WAIT_ABANDONED */
+		assert(0);
+		return false;
+	}
+	inline void MutexUnlock(MutexHandle& mutex)
+	{
+//		ReleaseSemaphore(mutex, 1, NULL);
+		ReleaseMutex(mutex);
+	}
+	inline void MutexTerm(MutexHandle& mutex) { CloseHandle(mutex); }
+#else
+	typedef pthread_mutex_t MutexHandle;
+	inline void MutexInit(MutexHandle& mutex)
+	{
+#if 1
+		pthread_mutex_init(&mutex, NULL);
+#else
+		pthread_mutexattr_t attr;
+		pthread_mutexattr_init(&attr);
+		if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_TIMED_NP)) {
+			perror("pthread_mutexattr_settype");
+			exit(1);
+		}
+		pthread_mutex_init(&mutex, &attr);
+		pthread_mutexattr_destroy(&attr);
+#endif
+	}
+	inline void MutexLock(MutexHandle& mutex) { pthread_mutex_lock(&mutex); }
+#if 0
+	inline bool MutexLockTimeout(MutexHandle& mutex, int msec)
+	{
+		timespec absTime;
+		clock_gettime(CLOCK_REALTIME, &absTime);
+		absTime.tv_sec += msec / 1000;
+		absTime.tv_nsec += msec % 1000;
+		bool ret = pthread_mutex_timedlock(&mutex, &absTime) == 0;
+		return ret;
+	}
+#endif
+	inline void MutexUnlock(MutexHandle& mutex) { pthread_mutex_unlock(&mutex); }
+	inline void MutexTerm(MutexHandle& mutex) { pthread_mutex_destroy(&mutex); }
+#endif
+
+template<class T>
+class AutoLockT {
+public:
+	explicit AutoLockT(T &t)
+		: t_(t)
+	{
+		t_.lock();
+	}
+	~AutoLockT()
+	{
+		t_.unlock();
+	}
+private:
+	T& t_;
+	AutoLockT& operator=(const AutoLockT&);
+};
+
+} // cybozu::thread
+
+class Mutex {
+	friend class cybozu::ConditionVariable;
+public:
+	Mutex()
+	{
+		thread::MutexInit(hdl_);
+	}
+	~Mutex()
+	{
+		thread::MutexTerm(hdl_);
+	}
+	void lock()
+	{
+		thread::MutexLock(hdl_);
+	}
+#if 0
+	bool lockTimeout(int msec)
+	{
+		return thread::MutexLockTimeout(hdl_, msec);
+	}
+#endif
+	void unlock()
+	{
+		thread::MutexUnlock(hdl_);
+	}
+private:
+	Mutex(const Mutex&);
+	Mutex& operator=(const Mutex&);
+	thread::MutexHandle hdl_;
+};
+
+typedef cybozu::thread::AutoLockT<cybozu::Mutex> AutoLock;
+
+} // cybozu

data/vendor/cybozulib/include/cybozu/nlp/mecab.hpp

@@ -0,0 +1,96 @@
+#pragma once
+/**
+	@file
+	@brief wrapper of MeCab
+
+	@author MITSUNARI Shigeo(@herumi)
+*/
+#include <string>
+#include <assert.h>
+#ifdef _WIN32
+#include <winsock2.h>
+#endif
+#include "mecab.h"
+#include <cybozu/exception.hpp>
+#ifdef _WIN32
+#pragma comment(lib, "libmecab.lib")
+#endif
+
+namespace cybozu { namespace nlp {
+
+struct Mecab {
+	Mecab(const char *option = "-O wakati")
+		: tagger_(MeCab::createTagger(option))
+		, node_(0)
+	{
+		if (tagger_ == 0) {
+			throw cybozu::Exception("nlp:mecab:createTagger");
+		}
+	}
+	/**
+		T must have push_back(std::string)
+	*/
+	template<class T>
+	bool parse(T& out, const char *str, size_t strLen = 0)
+	{
+		if (strLen == 0) {
+			strLen = strlen(str);
+		}
+		const char *p = tagger_->parse(str, strLen);
+		if (p == 0) return false;
+		while (*p) {
+			if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') {
+				p++;
+				continue;
+			}
+			const char *q = strchr(p, ' ');
+			if (q == 0) {
+				out.push_back(p);
+				break;
+			}
+			out.push_back(std::string(p, q));
+			p = q + 1;
+		}
+		return true;
+	}
+	void set(const char *str, size_t strLen = 0)
+	{
+		if (strLen == 0) {
+			strLen = strlen(str);
+		}
+		node_ = tagger_->parseToNode(str, strLen);
+	}
+	void set(const std::string& str)
+	{
+		set(&str[0], str.size());
+	}
+	bool isEnd() const
+	{
+		if (node_ == 0) return true;
+		return node_->stat == MECAB_EOS_NODE;
+	}
+	const char *getPos() const { return node_->surface; }
+	size_t getSize() const { return node_->length; }
+	/* adhoc */
+	bool isNoun() const
+	{
+		assert(node_);
+		const char *p = node_->feature;
+		if (node_->length < 2) return false;
+		return p[0] == '\xE5' && p[1] == '\x90' && p[2] == '\x8D';
+	}
+	void next()
+	{
+		assert(node_);
+		node_ = node_->next;
+	}
+	~Mecab()
+	{
+		delete tagger_;
+	}
+private:
+	MeCab::Tagger *tagger_;
+	const MeCab::Node *node_;
+};
+
+} } // cybozu::nlp

data/vendor/cybozulib/include/cybozu/nlp/plsi.hpp

@@ -0,0 +1,315 @@
+#pragma once
+/**
+	@file
+	@brief pLSI
+	@author MITSUNARI Shigeo(@herumi)
+*/
+
+#include <fstream>
+#include <map>
+#include <limits>
+#include <math.h>
+#include <cybozu/string_operation.hpp>
+#include <cybozu/time.hpp>
+#include <cybozu/nlp/random.hpp>
+#include <cybozu/nlp/sparse.hpp>
+#include <cybozu/nlp/top_score.hpp>
+
+namespace cybozu { namespace nlp {
+
+namespace local {
+
+template<class os, typename T>
+os& dump(os& out, const std::vector<T>& list) {
+	out << "{ ";
+	for (typename std::vector<T>::const_iterator i = list.begin(), ie = list.end(); i != ie; ++i) {
+		out << *i << " ";
+	}
+	out << "}";
+	return out;
+}
+
+} // local
+
+//const double NaN = std::numeric_limits<double>::quiet_NaN();
+
+typedef cybozu::nlp::SparseVector<bool> BoolSVec;
+typedef cybozu::nlp::SparseVector<double> DoubleSVec;
+typedef std::vector<BoolSVec> SMatrix;
+
+template<typename T>
+bool hasKey(const std::map<T, size_t>& map, T key) { return map.find(key) != map.end(); }
+
+
+class Plsi {
+public:
+	typedef int ITEM_TYPE;
+	typedef int USER_TYPE;
+
+	enum SEARCH_TYPE {
+		JOINT,
+		CONDITIONAL,
+		POSTERIOR
+	};
+private:
+	typedef std::vector<double> DoubleVec;
+	typedef std::vector<DoubleVec> DoubleVecVec;
+	std::map<USER_TYPE, size_t> users_;
+	std::vector<USER_TYPE> userlist_;
+
+	std::map<ITEM_TYPE, size_t> items_;
+	std::vector<ITEM_TYPE> itemlist_;
+
+	SMatrix matrix_; // item => users
+
+	// probability of p(z), p(x|z), p(y|z)
+	DoubleVec z_;
+	DoubleVecVec user_z_, item_z_;
+
+	template<class os>
+	friend os& dump(os& out, const Plsi& x) {
+		out << x.matrix_.size() << std::endl;
+		local::dump(out, x.z_) << std::endl;
+		return out;
+	}
+
+public:
+	size_t get_item_id(ITEM_TYPE item) {
+		if (hasKey(items_, item)) return items_[item];
+
+		size_t id = items_[item] = itemlist_.size();
+		itemlist_.push_back(item);
+		matrix_.push_back(BoolSVec());
+		return id;
+	}
+
+	BoolSVec& getItem(ITEM_TYPE item) {
+		return matrix_[get_item_id(item)];
+	}
+
+	size_t get_user_id(USER_TYPE user) {
+		if (hasKey(users_, user)) return users_[user];
+
+		size_t id = users_[user] = userlist_.size();
+		userlist_.push_back(user);
+		return id;
+	}
+
+	ITEM_TYPE get_item_key(size_t item_id) {
+		return itemlist_[item_id];
+	}
+
+	/**
+		@brief retrieve relevant items for query user
+	*/
+	cybozu::nlp::TopScore<size_t>::Table search_items(USER_TYPE user, int top = 10) {
+		int K = (int)z_.size();
+		size_t user_id = get_user_id(user);
+
+		double p_x = 0; // p(x) = sum p(z)p(x|z)
+		DoubleVec p_z_x; // p(z|x) = p(z)p(x|z) / p(x)
+		for (int k = 0; k < K; k++) {
+			double p = z_[k] * user_z_[k][user_id];
+			p_x += p;
+			p_z_x.push_back(p);
+		}
+
+		cybozu::nlp::TopScore<size_t> ranking(top);
+		for (size_t item_id = 0; item_id < items_.size(); item_id++) {
+			double score = 0; // p(y|x) = sum _z p(y|z) * p(z|x)
+			for (int k = 0; k < K; k++) {
+				score += item_z_[k][item_id] * p_z_x[k];
+			}
+			ranking.add(score / p_x, item_id);
+		}
+		return ranking.getTable();
+	}
+
+	/**
+		@brief retrieve similar items for query item
+	*/
+	cybozu::nlp::TopScore<size_t>::Table similar_items(ITEM_TYPE item, SEARCH_TYPE search_type, int top=10) {
+		int K = (int)z_.size();
+		size_t target_item_id = get_item_id(item);
+
+		cybozu::nlp::TopScore<size_t> ranking(top);
+		if (search_type == POSTERIOR) {
+			for (size_t item_id = 0; item_id < items_.size(); item_id++) {
+				// p(y1=target|y2=item_id) = sum _z p(target|z) * p(item_id|z) * p(z) / p(item_id)
+				double score = 0, p_y = 0;
+				for(int k=0;k<K;++k) {
+					double p = item_z_[k][item_id] * z_[k];
+					p_y += p;
+					score += item_z_[k][target_item_id] * p;
+				}
+
+				ranking.add(score / p_y, item_id);
+			}
+
+		} else if (search_type == CONDITIONAL) {
+			double p_y = 0;			  // p(y=target) = sum p(z)p(y=target|z)
+			DoubleVec p_z_y;   // p(z)p(y=target|z)
+			for (int k = 0; k < K; k++) {
+				double p = z_[k] * item_z_[k][target_item_id];
+				p_y += p;
+				p_z_y.push_back(p);
+			}
+			for (size_t item_id = 0; item_id < items_.size(); item_id++) {
+				// p(y1=item_id|y2=target) = sum _z p(y1|z) * p(z|y2) = sum _z p(y1|z) * p(y2|z) * p(z) / p(y2)
+				double score = 0;
+				for (int k = 0; k < K; k++) {
+					score += item_z_[k][item_id] * p_z_y[k];
+				}
+
+				ranking.add(score / p_y, item_id);
+			}
+
+		} else if (search_type == JOINT) {
+			for (size_t item_id = 0; item_id < items_.size(); item_id++) {
+				// p(y1=item_id, y2=i) = sum _z p(y1|z) * p(y2|z) * p(z)
+				double score = 0;
+				for (int k = 0; k < K; k++) {
+					score += item_z_[k][item_id] * item_z_[k][target_item_id] * z_[k];
+				}
+				ranking.add(score, item_id);
+			}
+		}
+		return ranking.getTable();
+	}
+
+	/**
+		@brief calcurate perplexity
+	*/
+	double perplexity()
+	{
+		int K = (int)z_.size();
+
+		// p(x) = sum p(z)p(x|z)
+		DoubleVec p_x;
+		for (size_t user_id = 0; user_id < users_.size(); user_id++) {
+			double p = 0;
+			for (int k = 0; k < K; k++) {
+				p += z_[k] * user_z_[k][user_id];
+			}
+			p_x.push_back(p);
+		}
+
+		int denom = 0;
+		double sum = 0;
+		for (size_t item_id = 0; item_id < matrix_.size(); item_id++) {
+			BoolSVec& item_users = matrix_[item_id];
+			for (BoolSVec::const_iterator i = item_users.begin(), ie = item_users.end(); i != ie; ++i) {
+				++denom;
+				size_t user_id = i.pos();
+
+				// p(y|x) = sum p(y|z)p(z|x) = sum p(y|z)p(x|z)p(z)/p(x)
+				double p = 0;
+				for (int k = 0; k < K; k++) {
+					p += z_[k] * user_z_[k][user_id] * item_z_[k][item_id];
+				}
+				sum += log(p / p_x[user_id]);
+			}
+		}
+		return exp(-sum/denom);
+	}
+
+	/**
+		@brief start learning (initialize learning)
+	*/
+	void startLearning(int K)
+	{
+		size_t M = users_.size();
+		size_t N = items_.size();
+		user_z_.resize(K);
+		item_z_.resize(K);
+		cybozu::nlp::UniformRandomGenerator rand(0.25, 0.75);
+		for (int k = 0; k < K; k++) {
+			// initialize p(z=k)
+			z_.push_back(1.0/K);
+
+			// initialize p(x=user|z=k)
+			DoubleVec& uvec = user_z_[k];
+			for (size_t j = 0; j < M; j++) uvec.push_back(1.0/M);
+
+			// initialize p(y=item|z=k)
+			DoubleVec& ivec = item_z_[k];
+			double s = 0;
+			for (size_t j = 0; j < N; j++) {
+				double r = rand.getDouble();
+				ivec.push_back(r);
+				s += r;
+			}
+			for(size_t j = 0; j < N; j++) ivec[j] /= s;
+		}
+
+	}
+
+	/**
+		@brief step learning (called repeatedly after initialization learning)
+		@param[in] beta  temperature for tempered EM
+		@return likelyhood for previous iteration
+	*/
+	double step(double beta = 1)
+	{
+		int K = (int)z_.size();
+
+		DoubleVec z_numer;
+		DoubleVecVec user_numer, item_numer;
+		z_numer.resize(K);
+		user_numer.resize(K);
+		item_numer.resize(K);
+		for (int k = 0; k < K; k++) {
+			user_numer[k].resize(users_.size());
+			item_numer[k].resize(items_.size());
+		}
+		int denom = 0;
+		double likelihood = 0;
+		DoubleVec p_z_xy;
+		p_z_xy.resize(K);
+
+		for (size_t item_id = 0; item_id < matrix_.size(); ++item_id) {
+			BoolSVec& item_users = matrix_[item_id];
+			for (BoolSVec::const_iterator i = item_users.begin(), ie = item_users.end(); i != ie; ++i) {
+				// when n(x, y) = 1(true)
+				++denom;
+				size_t user_id = i.pos();
+
+				// E-step: p(z|x,y)
+				double sum = 0;
+				for (int k = 0; k < K; k++) {
+					// p(z=k)p(x=user_id|z=k)p(y=item_id|z=k)
+					double p = pow(z_[k] * user_z_[k][user_id] * item_z_[k][item_id], beta);
+					p_z_xy[k] = p;
+					sum += p;
+				}
+
+				// normalize & M-step
+				for (int k = 0; k < K; k++) {
+					double p = p_z_xy[k] / sum;
+
+					user_numer[k][user_id] += p;
+					item_numer[k][item_id] += p;
+					z_numer[k] += p;
+				}
+				likelihood += log(sum);
+			}
+		}
+
+		// M-step: update
+		for (int k = 0; k < K; k++) {
+			double z_num = z_numer[k];
+			z_[k] = z_num / denom;
+			for (size_t item_id = 0; item_id < items_.size(); ++item_id) {
+				item_z_[k][item_id] = item_numer[k][item_id] / z_num;
+			}
+			for (size_t user_id = 0; user_id < users_.size(); ++user_id) {
+				user_z_[k][user_id] = user_numer[k][user_id] / z_num;
+			}
+		}
+
+		// log-likelihood of previous iteration
+		return likelihood;
+	}
+};
+
+} } // cybozu::nlp

data/vendor/cybozulib/include/cybozu/nlp/random.hpp

@@ -0,0 +1,74 @@
+#pragma once
+/**
+	@file
+	@brief normal random generator
+
+	@author MITSUNARI Shigeo(@herumi)
+	@author MITSUNARI Shigeo
+*/
+#include <cybozu/xorshift.hpp>
+
+namespace cybozu { namespace nlp {
+
+/*
+	use xor shift
+*/
+class UniformRandomGenerator {
+	double a_;
+	double b_;
+	cybozu::XorShift rg;
+public:
+	/* generate uniform random value in [a, b) */
+	explicit UniformRandomGenerator(double a = 0, double b = 1, int seed = 0)
+		: a_(a)
+		, b_(b)
+		, rg(seed)
+	{
+	}
+	void init(int seed = 0)
+	{
+		rg.init(seed);
+	}
+	/* [0, 2^32) random number */
+	uint32_t operator()() { return rg.get32(); }
+	uint32_t get32() { return rg.get32(); }
+	uint64_t get64() { return rg.get64(); }
+	/* [a, b) random number */
+	double getDouble()
+	{
+		uint32_t x = get32() >> 5;
+		uint32_t y = get32() >> 6;
+		double z = (x * double(1U << 26) + y) * (1.0 / double(1LL << 53));
+		return (b_ - a_) * z + a_;
+	}
+};
+
+/*
+	normal random generator
+*/
+class NormalRandomGenerator {
+	UniformRandomGenerator gen_;
+	double u_;
+	double s_;
+public:
+	explicit NormalRandomGenerator(double u = 0, double s = 1, int seed = 0)
+		: gen_(seed)
+		, u_(u)
+		, s_(s)
+	{
+	}
+	void init(int seed = 0)
+	{
+		gen_.init(seed);
+	}
+	double get()
+	{
+		double sum = -6;
+		for (int i = 0; i < 12; i++) {
+			sum += gen_.getDouble();
+		}
+		return sum * s_ + u_;
+	}
+};
+
+} } // cybozu::nlp