ooxml_crypt 0.1.0

data/vendor/cybozulib/include/cybozu/fmindex.hpp

@@ -0,0 +1,291 @@
+#pragma once
+/**
+	@file
+	@brief FM-index
+	@author MITSUNARI Shigeo(@herumi)
+	@license modified new BSD license
+	http://opensource.org/licenses/BSD-3-Clause
+*/
+#include <map>
+#include <vector>
+#include <fstream>
+#include <stdio.h>
+#ifdef CYBOZU_FMINDEX_USE_CSUCVECTOR
+	#include <cybozu/csucvector.hpp>
+#endif
+#include <cybozu/wavelet_matrix.hpp>
+#include <cybozu/bitvector.hpp>
+#include <cybozu/frequency.hpp>
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable:4244)
+#pragma warning(disable:4389)
+#pragma warning(disable:4018)
+#endif
+#include "sais.hxx"
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable:4127) // constant condition
+#endif
+
+namespace cybozu {
+/*
+	T : type of alphabet
+	isRawData : deal with input data as is
+	T must be uint8_t or uint16_t if isRawData
+*/
+template<class T, bool isRawData = false>
+class FMindexT {
+public:
+	static const size_t maxCharNum = size_t(1) << (sizeof(T) * 8);
+	typedef std::vector<uint32_t> Vec32;
+	typedef std::vector<T> Vec;
+#ifdef CYBOZU_FMINDEX_USE_CSUCVECTOR
+	typedef cybozu::CSucVector SucVector;
+#else
+	typedef cybozu::SucVectorT<uint32_t, false> SucVector;
+#endif
+	typedef cybozu::WaveletMatrixT<false, SucVector> WaveletMatrix;
+	Vec32 cf;
+	WaveletMatrix wm;
+	Vec32 alignedSa;
+	SucVector alignedPos;
+	cybozu::Frequency<T, uint32_t> freq;
+	int skip_;
+	size_t charNum_;
+
+	/*
+		setup freq, cf by [begin, end)
+	*/
+	template<class Iter>
+	void initCf(Vec& v, Iter begin, Iter end)
+	{
+		const size_t size = std::distance(begin, end);
+		if (size >= (uint64_t(1) << 32) - 1) {
+			throw cybozu::Exception("FMindexT:initCf:too large dataSize") << size;
+		}
+		v.resize(size + 1); // add NUL at the end of data
+		if (isRawData) {
+			assert(sizeof(T) <= 16);
+			charNum_ = size_t(1) << (sizeof(T) * 8);
+			std::vector<uint32_t> charNumTbl(charNum_);
+			charNumTbl[0] = 1;
+			for (size_t i = 0; i < size; i++) {
+				T c = *begin++;
+				if (c <= 0) throw cybozu::Exception("FMindext:initCf:zero alphabet") << c;
+				v[i] = c;
+				charNumTbl[c]++;
+			}
+			cf.resize(charNum_);
+			uint32_t sum = 0;
+			for (size_t i = 0; i < charNum_; i++) {
+				cf[i] = sum;
+				sum += charNumTbl[i];
+			}
+		} else {
+			freq.init(begin, end);
+			charNum_ = freq.size() + 1; // +1 means last zero
+			if (charNum_ > maxCharNum) throw cybozu::Exception("FMindexT:initCf:too many alphabet");
+			for (size_t i = 0; i < size; i++) {
+				v[i] = static_cast<T>(freq.getIndex(*begin++) + 1);
+			}
+			cf.resize(charNum_);
+			cf[0] = 0;
+			uint32_t sum = 1;
+			for (size_t i = 1; i < charNum_; i++) {
+				cf[i] = sum;
+				sum += freq.getFrequency(freq.getElement(i - 1));
+			}
+		}
+	}
+	void initBwt(Vec& bwt, const Vec& s, const Vec32& sa) const
+	{
+		const size_t size = sa.size();
+		bwt.resize(size);
+		for (size_t i = 0; i < size; i++) {
+			if (sa[i] > 0) {
+				bwt[i] = s[sa[i] - 1];
+			} else {
+				bwt[i] = s[size - 1];
+			}
+		}
+	}
+	size_t getBitLen(size_t x) const
+	{
+		if (x == 0) return 1;
+		size_t ret = 0;
+		while (x > 0) {
+			x >>= 1;
+			ret++;
+		}
+		return ret;
+	}
+public:
+	FMindexT()
+		: skip_(8)
+		, charNum_(0)
+	{
+	}
+
+	/*
+		[begin, end)
+		replace '\0' in [begin, end) with space
+		append '\0' at the end of [begin, end)
+	*/
+	template<class Iter>
+	void init(Iter begin, Iter end, int skip = 8)
+	{
+		if (skip <= 0) {
+			throw cybozu::Exception("FMindexT:buildFMindex:skip is positive") << skip;
+		}
+		skip_ = skip;
+		Vec v;
+		initCf(v, begin, end);
+		const size_t dataSize = v.size();
+
+		Vec32 sa;
+		sa.resize(dataSize);
+		if (saisxx(&v[0], &sa[0], (int)dataSize, (int)charNum_) == -1) {
+			throw cybozu::Exception("FMindexT:init:saisxx");
+		}
+		Vec bwt;
+		initBwt(bwt, v, sa);
+		wm.init(bwt, getBitLen(charNum_));
+
+#if 1
+		cybozu::BitVector bv;
+		bv.resize(dataSize);
+		for (size_t i = 0; i < dataSize; i++) {
+			if ((sa[i] % skip) == 0) {
+				bv.set(i);
+				alignedSa.push_back(sa[i]);
+			}
+		}
+		alignedPos.init(bv.getBlock(), bv.size());
+#else
+		alignedPos.resize(dataSize);
+		for (size_t i = 0; i < dataSize; i++) {
+			if ((sa[i] % skip) == 0) {
+				alignedPos.set(i);
+				alignedSa.push_back(sa[i]);
+			}
+		}
+		alignedPos.ready();
+#endif
+	}
+
+	/*
+		get range of bwt for key
+	*/
+	template<class Int, class Key>
+	bool getRange(Int* pbegin, Int* pend, const Key& _key) const
+	{
+		if (_key.empty()) return false;
+		const size_t keySize = _key.size();
+		const typename Key::value_type *key;
+		Key cvtKey;
+		if (isRawData) {
+			key = &_key[0];
+		} else {
+			cvtKey.resize(keySize);
+			for (size_t i = 0; i < keySize; i++) {
+				if (freq.getFrequency(_key[i]) == 0) return false;
+				cvtKey[i] = typename Key::value_type(freq.getIndex(_key[i]) + 1);
+			}
+			key = &cvtKey[0];
+		}
+		size_t i = keySize - 1;
+		size_t begin = 0;
+		size_t end = wm.size();
+		while (begin < end) {
+			const T c = key[i];
+			const uint32_t cfc = cf[c];
+			begin = cfc + wm.rank(c, begin);
+			end = cfc + wm.rank(c, end);
+			if (i == 0) break;
+			i--;
+		}
+
+		if (begin < end) {
+			*pbegin = Int(begin);
+			*pend = Int(end);
+			return true;
+		}
+		return false;
+	}
+	template<class Int>
+	bool getRange(Int* pbegin, Int* pend, const char *key) const
+	{
+		return getRange(pbegin, pend, std::string(key));
+	}
+	size_t convertPosition(size_t bwtPos) const
+	{
+		size_t t = 0;
+		while (!alignedPos.get(bwtPos)) {
+			T c;
+			bwtPos = wm.get(&c, bwtPos);
+			bwtPos += cf[c];
+			t++;
+		}
+		return t + alignedSa[alignedPos.rank1(bwtPos)];
+	}
+	/*
+		get previous string at pos
+		@note assume T is vector or std::string
+	*/
+	template<class Str>
+	void getPrevString(Str& str, size_t bwtPos, size_t len) const
+	{
+		str.resize(len);
+		T c;
+		while (len > 0) {
+			bwtPos = wm.get(&c, bwtPos);
+			bwtPos += cf[c];
+			if (c == 0) {
+				str.erase(str.begin(), str.begin() + len);
+				return;
+			}
+			len--;
+			str[len] = isRawData ? c : freq.getElement(c - 1);
+		}
+	}
+
+	template<class OutputStream>
+	void save(OutputStream& os) const
+	{
+		cybozu::save(os, skip_);
+		cybozu::savePodVec(os, cf);
+		wm.save(os);
+		cybozu::savePodVec(os, alignedSa);
+		alignedPos.save(os);
+		if (!isRawData) freq.save(os);
+	}
+	template<class InputStream>
+	void load(InputStream& is)
+	{
+		cybozu::load(skip_, is);
+		cybozu::loadPodVec(cf, is);
+		wm.load(is);
+		cybozu::loadPodVec(alignedSa, is);
+		alignedPos.load(is);
+		if (isRawData) {
+			charNum_ = size_t(1) << (sizeof(T) * 8);
+		} else {
+			freq.load(is);
+			charNum_ = freq.size();
+		}
+	}
+};
+
+typedef FMindexT<uint8_t> FMindex;
+
+} // cybozu
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif

data/vendor/cybozulib/include/cybozu/format.hpp

@@ -0,0 +1,93 @@
+#pragma once
+/**
+	@file
+	@brief format string
+	@author MITSUNARI Shigeo(@herumi)
+*/
+#include <string>
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <cybozu/exception.hpp>
+
+#if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 4)
+#define CYBOZU_FORMAT_DISABLE_WARNING
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-nonliteral"
+#endif
+
+namespace cybozu {
+
+inline void vformat(std::string& str, const char *format, va_list args)
+{
+#ifdef _MSC_VER
+	_locale_t curLoc = _get_current_locale();
+	int size = _vscprintf_l(format, curLoc, args);
+	if (size < 0 || size >= INT_MAX) throw cybozu::Exception("vformat:_vscprintf_l");
+
+	str.resize(size + 1);
+
+	int ret = _vsprintf_s_l(&str[0], size + 1, format, curLoc, args);
+	if (ret < 0) throw cybozu::Exception("vformat:_vsprintf_s_l");
+	str.resize(size);
+#else
+#if 1
+	char *p;
+	int ret = vasprintf(&p, format, args);
+	if (ret < 0) throw cybozu::Exception("vformat:vasnprintf");
+	try {
+		str.assign(p, ret);
+		free(p);
+	} catch (...) {
+		free(p);
+		throw std::bad_alloc();
+	}
+#else
+	// slow
+	va_list keep;
+	va_copy(keep, args);
+	int len = vsnprintf(0, 0, format, args); // len excludes the null byte
+	if (len < 0) throw cybozu::Exception("vformat:vasnprintf err1");
+	str.resize(len + 1);
+	len = vsnprintf(&str[0], str.size(), format, keep); // len incluedes the null byte
+	if (len < 0) throw cybozu::Exception("vformat:vasnprintf err2");
+	str.resize(len);
+#endif
+#endif
+}
+
+#ifdef _MSC_VER
+	#define CYBOZU_FORMAT_PRINTF _Printf_format_string_
+#else
+	#define CYBOZU_FORMAT_PRINTF
+#endif
+
+#ifdef __GNUC__
+__attribute__((format(printf, 2, 3)))
+#endif
+inline void format(std::string& str, CYBOZU_FORMAT_PRINTF const char *format, ...)
+{
+	va_list args;
+	va_start(args, format);
+	cybozu::vformat(str, format, args);
+	va_end(args);
+}
+
+#ifdef __GNUC__
+__attribute__((format(printf, 1, 2)))
+#endif
+inline std::string format(CYBOZU_FORMAT_PRINTF const char *format, ...)
+{
+	std::string str;
+	va_list args;
+	va_start(args, format);
+	cybozu::vformat(str, format, args);
+	va_end(args);
+	return str;
+}
+
+} // cybozu
+
+#ifdef CYBOZU_FORMAT_DISABLE_WARNING
+#pragma GCC diagnostic push
+#endif

data/vendor/cybozulib/include/cybozu/frequency.hpp

@@ -0,0 +1,264 @@
+#pragma once
+/**
+	@file
+	@brief frequency of elements in a sequence
+	@author MITSUNARI Shigeo(@herumi)
+	@license modified new BSD license
+	http://opensource.org/licenses/BSD-3-Clause
+*/
+#include <assert.h>
+#include <vector>
+#include <algorithm>
+#include <functional>
+#include <iostream>
+#include <cybozu/exception.hpp>
+#include <cybozu/unordered_map.hpp>
+#include <cybozu/serializer.hpp>
+
+namespace cybozu {
+
+namespace freq_local {
+
+template<class Element, class Int = size_t>
+class FrequencyVec {
+	static const size_t N = size_t(1) << (sizeof(Element) * 8);
+	size_t size_;
+	Int freqTbl_[N];
+	uint8_t char2idx_[N];
+	uint8_t idx2char_[N];
+	struct Greater {
+		const Int *p_;
+		explicit Greater(const Int *p) : p_(p) {}
+		bool operator()(uint8_t lhs, uint8_t rhs) const
+		{
+			Int a = p_[lhs];
+			Int b = p_[rhs];
+			if (a > b) return true;
+			if (a < b) return false;
+			return a > b;
+		}
+	};
+public:
+	typedef Element value_type;
+	typedef Int size_type;
+
+	FrequencyVec() { clear(); }
+	template<class Iter>
+	FrequencyVec(Iter begin, Iter end)
+	{
+		clear();
+		init(begin, end);
+	}
+	void clear()
+	{
+		size_ = 0;
+		memset(freqTbl_, 0, sizeof(freqTbl_));
+	}
+	template<class Iter>
+	void init(Iter begin, Iter end)
+	{
+		while (begin != end) {
+			append(*begin);
+			++begin;
+		}
+		ready();
+	}
+	void append(const Element e)
+	{
+		freqTbl_[uint8_t(e)]++;
+	}
+	void ready()
+	{
+		for (size_t i = 0; i < N; i++) idx2char_[i] = uint8_t(i);
+		Greater greater(freqTbl_);
+		std::sort(idx2char_, idx2char_ + N, greater);
+		size_ = 0;
+		for (size_t i = 0; i < N; i++) {
+			uint8_t c = idx2char_[i];
+			char2idx_[c] = (uint8_t)i;
+			if (freqTbl_[c]) size_++;
+		}
+	}
+	/*
+		element -> freq
+	*/
+	Int getFrequency(Element e) const { return freqTbl_[uint8_t(e)]; }
+	/*
+		element -> idx
+	*/
+	Int getIndex(Element e) const { return char2idx_[uint8_t(e)]; }
+	/*
+		idx -> element
+	*/
+	Element getElement(size_t idx) const
+	{
+//		if (idx >= N) throw cybozu::Exception("Frequency:getElement:bad idx") << idx;
+		assert(idx < N);
+		return Element(idx2char_[idx]);
+	}
+	size_t size() const { return size_; }
+	template<class InputStream>
+	void load(InputStream& is)
+	{
+		cybozu::load(size_, is);
+		cybozu::loadRange(freqTbl_, N, is);
+		cybozu::loadRange(char2idx_, N, is);
+		cybozu::loadRange(idx2char_, N, is);
+	}
+	void save(std::ostream& os) const
+	{
+		cybozu::save(os, size_);
+		cybozu::saveRange(os, freqTbl_, N);
+		cybozu::saveRange(os, char2idx_, N);
+		cybozu::saveRange(os, idx2char_, N);
+	}
+	void put() const
+	{
+		for (size_t i = 0; i < size_; i++) {
+			uint8_t c = idx2char_[i];
+			printf("%d %d %d\n", (int)i, c, freqTbl_[c]);
+		}
+	}
+};
+
+} // cybozu::freq_local
+
+/*
+	count Element
+	Element : type of element
+	Int : type of counter
+*/
+template<class Element, class Int = size_t>
+class Frequency {
+	struct FreqIdx {
+		Int freq;
+		mutable Int idx;
+		template<class InputStream>
+		void load(InputStream& is)
+		{
+			cybozu::load(freq, is);
+			cybozu::load(idx, is);
+		}
+		template<class OutputStream>
+		void save(OutputStream& os) const
+		{
+			cybozu::save(os, freq);
+			cybozu::save(os, idx);
+		}
+	};
+	typedef CYBOZU_NAMESPACE_STD::unordered_map<Element, FreqIdx> Map;
+	typedef Element value_type;
+	typedef Int size_type;
+	typedef std::vector<typename Map::const_iterator> Idx2Ref;
+	static inline bool greater(typename Map::const_iterator i, typename Map::const_iterator j)
+	{
+		const Int a = i->second.freq;
+		const Int b = j->second.freq;
+		if (a > b) return true;
+		if (a < b) return false;
+		return i->first > j->first;
+	}
+	Map m_;
+	Idx2Ref idx2ref_;
+	void initIdx2Ref()
+	{
+		idx2ref_.resize(m_.size());
+		size_t pos = 0;
+		for (typename Map::const_iterator i = m_.begin(), ie = m_.end(); i != ie; ++i) {
+			idx2ref_[pos++] = i;
+		}
+		std::sort(idx2ref_.begin(), idx2ref_.end(), greater);
+	}
+public:
+	Frequency(){ clear(); }
+	template<class Iter>
+	Frequency(Iter begin, Iter end)
+	{
+		clear();
+		init(begin, end);
+	}
+	void clear()
+	{
+		m_.clear();
+		idx2ref_.clear();
+	}
+	template<class Iter>
+	void init(Iter begin, Iter end)
+	{
+		while (begin != end) {
+			append(*begin);
+			++begin;
+		}
+		ready();
+	}
+	void append(const Element& e)
+	{
+		m_[e].freq++;
+	}
+	void ready()
+	{
+		initIdx2Ref();
+		for (size_t i = 0, ie = idx2ref_.size(); i < ie; i++) {
+			idx2ref_[i]->second.idx = (Int)i;
+		}
+	}
+	/*
+		element -> freq
+	*/
+	Int getFrequency(const Element& e) const
+	{
+		typename Map::const_iterator i = m_.find(e);
+		return (i != m_.end()) ? i->second.freq : 0;
+	}
+	/*
+		element -> idx
+	*/
+	Int getIndex(const Element& e) const
+	{
+		typename Map::const_iterator i = m_.find(e);
+		if (i == m_.end()) throw cybozu::Exception("Frequency:getIndex:not found") << e;
+		return i->second.idx;
+	}
+	/*
+		idx -> element
+	*/
+	const Element& getElement(size_t idx) const
+	{
+		if (idx >= idx2ref_.size()) throw cybozu::Exception("Frequency:getElement:bad idx") << idx;
+		return idx2ref_[idx]->first;
+	}
+	size_t size() const { return idx2ref_.size(); }
+	template<class InputStream>
+	void load(InputStream& is)
+	{
+		cybozu::load(m_, is);
+		initIdx2Ref();
+	}
+	template<class OutputStream>
+	void save(OutputStream& os) const
+	{
+		cybozu::save(os, m_);
+	}
+	void put() const
+	{
+		for (size_t i = 0, n = idx2ref_.size(); i < n; i++) {
+			typename Map::const_iterator j = idx2ref_[i];
+			std::cout << i << ' ' << j->first << ' ' << j->second.freq << std::endl;
+		}
+	}
+};
+
+template<class Int>
+struct Frequency<uint8_t, Int> : freq_local::FrequencyVec<uint8_t, Int> {
+	Frequency() {}
+	template<class Iterator>
+	Frequency(Iterator begin, Iterator end) : freq_local::FrequencyVec<uint8_t, Int>(begin, end) {}
+};
+template<class Int>
+struct Frequency<char, Int> : freq_local::FrequencyVec<char, Int> {
+	Frequency() {}
+	template<class Iterator>
+	Frequency(Iterator begin, Iterator end) : freq_local::FrequencyVec<char, Int>(begin, end) {}
+};
+
+} // cybozu

data/vendor/cybozulib/include/cybozu/hash.hpp

@@ -0,0 +1,67 @@
+#pragma once
+#include <cybozu/inttype.hpp>
+
+namespace cybozu {
+
+template<class Iter>
+uint32_t hash32(Iter begin, Iter end, uint32_t v = 0)
+{
+	if (v == 0) v = 2166136261U;
+	while (begin != end) {
+		v ^= *begin++;
+		v *= 16777619;
+	}
+	return v;
+}
+template<class Iter>
+uint64_t hash64(Iter begin, Iter end, uint64_t v = 0)
+{
+	if (v == 0) v = 14695981039346656037ULL;
+	while (begin != end) {
+		v ^= *begin++;
+		v *= 1099511628211ULL;
+	}
+	v ^= v >> 32;
+	return v;
+}
+template<class T>
+uint32_t hash32(const T *x, size_t n, uint32_t v = 0)
+{
+	return hash32(x, x + n, v);
+}
+template<class T>
+uint64_t hash64(const T *x, size_t n, uint64_t v = 0)
+{
+	return hash64(x, x + n, v);
+}
+
+} // cybozu
+
+namespace boost {
+
+template<class T>
+struct hash;
+
+} // boost
+
+#if CYBOZU_CPP_VERSION >= CYBOZU_CPP_VERSION_CPP11
+#include <functional>
+#else
+
+namespace std { CYBOZU_NAMESPACE_TR1_BEGIN
+
+#ifdef _MSC_VER
+	#pragma warning(push)
+	#pragma warning(disable : 4099) // missmatch class and struct
+#endif
+#if !(defined(__APPLE__) && defined(__clang__))
+template<class T>
+struct hash;
+#endif
+#ifdef _MSC_VER
+	#pragma warning(pop)
+#endif
+
+CYBOZU_NAMESPACE_TR1_END } // std
+
+#endif