ooxml_crypt 0.1.0

data/vendor/cybozulib/sample/exception_smpl.cpp

@@ -0,0 +1,46 @@
+#include <cybozu/exception.hpp>
+#include <cybozu/string.hpp>
+#include <cybozu/atoi.hpp>
+#include <iostream>
+#include <assert.h>
+#include <cybozu/stacktrace.hpp>
+
+struct MailException : cybozu::Exception {
+	MailException() : cybozu::Exception("mail") { }
+};
+
+void f2()
+{
+	const char *msg = "HTTP/...";
+	std::string abc = "abc";
+	char c = 'x';
+	int port = 80;
+	unsigned int s = 90;
+	MailException e;
+	e << "can't send" << msg << abc << c << port << s << '\n';
+	cybozu::StackTrace st;
+	e << st;
+	throw e;
+}
+
+void f1()
+{
+	f2();
+}
+
+void f0()
+{
+	f1();
+}
+
+int main()
+{
+	try {
+		f0();
+	} catch (cybozu::Exception &e) {
+		std::cout << "for user" << std::endl;
+		std::cout << e.toString() << std::endl;
+	} catch (...) {
+		std::cout << "Error!" << std::endl;
+	}
+}

data/vendor/cybozulib/sample/fmindex_smpl.cpp

@@ -0,0 +1,231 @@
+#include <fstream>
+#include <cybozu/time.hpp>
+#include <cybozu/fmindex.hpp>
+#include <cybozu/mmap.hpp>
+#include <cybozu/string.hpp>
+#include <cybozu/hash.hpp>
+#include <cybozu/benchmark.hpp>
+#include <set>
+
+#ifdef USE_UTF32
+typedef cybozu::FMindexT<cybozu::Char> FMindex;
+typedef cybozu::String String;
+#else
+typedef cybozu::FMindex FMindex;
+typedef std::string String;
+#endif
+
+typedef std::set<int> Set;
+
+void putSet(const Set& set)
+{
+	for (Set::const_iterator i = set.begin(), ie = set.end(); i != ie; ++i) {
+		std::cout << *i << ' ';
+	}
+	std::cout << std::endl;
+}
+
+template<class STRING>
+void simpleSearch(const std::string& inName, const std::string& queryFile, bool putHash)
+{
+	cybozu::Mmap m(inName);
+	STRING text(m.get(), m.size());
+
+	double beginTime = cybozu::GetCurrentTimeSec();
+
+	std::ifstream qs(queryFile.c_str(), std::ios::binary);
+	STRING key;
+	uint64_t hash = 0;
+	while (qs >> key) {
+		if (!putHash) std::cout << "query " << key << std::endl;
+		size_t p = 0;
+		Set set;
+		for (;;) {
+			size_t q = text.find(key, p);
+			if (q == std::string::npos) break;
+			set.insert((int)q);
+			p = q + 1;
+		}
+		if (putHash) {
+			hash = cybozu::hash64(set.begin(), set.end(), hash);
+		} else {
+			putSet(set);
+		}
+	}
+	if (putHash) printf("hash=%llx\n", (long long)hash);
+
+	double endTime = cybozu::GetCurrentTimeSec();
+	fprintf(stderr, "time: %gsec\n", endTime - beginTime);
+}
+
+template<class FMINDEX, class STRING>
+void recover(const std::string& inName, const std::string& outName)
+{
+	std::ifstream is(inName.c_str(), std::ios::binary);
+	FMINDEX f;
+	f.load(is);
+
+	double beginTime = cybozu::GetCurrentTimeSec();
+
+	STRING str;
+	f.getPrevString(str, 0, f.wm.size() - 1);
+	double endTime = cybozu::GetCurrentTimeSec();
+	fprintf(stderr, "time: %gsec\n", endTime - beginTime);
+	std::ofstream os(outName.c_str(), std::ios::binary);
+	os << str;
+}
+
+template<class FMINDEX, class STRING>
+void search(const std::string& inName, const std::string& queryFile, bool putHash, bool bench)
+{
+	std::ifstream is(inName.c_str(), std::ios::binary);
+	FMINDEX f;
+	f.load(is);
+
+	double beginTime = cybozu::GetCurrentTimeSec();
+
+	std::ifstream qs(queryFile.c_str(), std::ios::binary);
+	STRING key;
+	uint64_t hash = 0;
+	cybozu::CpuClock clkRange;
+	cybozu::CpuClock clkPos;
+	while (qs >> key) {
+		if (!putHash) std::cout << "query " << key << std::endl;
+		size_t begin, end = 0;
+		if (bench) clkRange.begin();
+		bool found = f.getRange(&begin, &end, key);
+		if (bench) clkRange.end();
+		Set set;
+		if (found) {
+			while (begin != end) {
+				if (bench) clkPos.begin();
+				int pos = (int)f.convertPosition(begin);
+				if (bench) clkPos.end();
+				set.insert(pos);
+				begin++;
+			}
+		}
+		if (putHash) {
+			hash = cybozu::hash64(set.begin(), set.end(), hash);
+		} else {
+			putSet(set);
+		}
+	}
+	if (putHash) printf("hash=%llx\n", (long long)hash);
+
+	double endTime = cybozu::GetCurrentTimeSec();
+	fprintf(stderr, "time: %gsec\n", endTime - beginTime);
+	if (bench) {
+		int rangeNum = (int)clkRange.getCount();
+		int posNum = (int)clkPos.getCount();
+		fprintf(stderr, "getRange %.2f(%d) pos %.2f(%d)\n", clkRange.getClock() / double(rangeNum), rangeNum, clkPos.getClock() / double(posNum), posNum);
+	}
+}
+
+template<class FMINDEX, class STRING>
+static void create(const std::string& inName, const std::string& outName, int skip)
+{
+	fprintf(stderr, "inName=%s, outName=%s, skip=%d\n", inName.c_str(), outName.c_str(), skip);
+
+	double beginTime = cybozu::GetCurrentTimeSec();
+
+	cybozu::Mmap m(inName);
+	FMINDEX f;
+	STRING text(m.get(), m.get() + m.size());
+	f.init(text.begin(), text.end(), skip);
+
+	double endTime = cybozu::GetCurrentTimeSec();
+	fprintf(stderr, "create time %gsec\n", endTime - beginTime);
+	std::ofstream os(outName.c_str(), std::ios::binary);
+	f.save(os);
+}
+
+void usage()
+{
+	printf("fmindex_smpl.exe (-c|-s|-r|-ss) file1 file2 [-skip skip][-hash][-time]\n");
+	printf(" -c : create index file\n");
+	printf("  file1 : any UTF-8 string file\n");
+	printf("  file2 : output index file\n");
+	printf("  -skip skip : skip to sampling(default 8)\n");
+	printf("  -hash : put position hash\n");
+	printf("  -time : benchmark\n");
+	printf(" -s : search mode\n");
+	printf("  file1 : index file\n");
+	printf("  file2 : query string file\n");
+	printf(" -r : recover mode\n");
+	printf("  file1 : index file\n");
+	printf("  file2 : org index file\n");
+	printf(" -ss: simple search\n");
+	printf("  file1 : any UTF-8 string file\n");
+	printf("  file2 : query string file\n");
+	exit(1);
+}
+
+int main(int argc, char* argv[])
+	try
+{
+	argc--, argv++;
+	std::string fName1;
+	std::string fName2;
+	std::string mode;
+	int skip = 8;
+	bool putHash = false;
+	bool bench = false;
+
+	while (argc > 0) {
+		if (strcmp(*argv, "-c") == 0) {
+			mode = *argv;
+		} else
+		if (strcmp(*argv, "-s") == 0) {
+			mode = *argv;
+		} else
+		if (strcmp(*argv, "-r") == 0) {
+			mode = *argv;
+		} else
+		if (strcmp(*argv, "-ss") == 0) {
+			mode = *argv;
+		} else
+		if (argc > 1 && strcmp(*argv, "-skip") == 0) {
+			argc--, argv++;
+			skip = atoi(*argv);
+		} else
+		if (strcmp(*argv, "-hash") == 0) {
+			putHash = true;
+		} else
+		if (strcmp(*argv, "-time") == 0) {
+			bench = true;
+		} else
+		if (**argv != '-' && fName1.empty()) {
+			fName1 = *argv;
+		} else
+		if (**argv != '-' && fName2.empty()) {
+			fName2 = *argv;
+		} else
+		{
+			usage();
+		}
+		argc--, argv++;
+	}
+	if (fName1.empty() || fName2.empty() || mode.empty()) {
+		usage();
+	}
+	if (mode == "-c") {
+		create<FMindex, String>(fName1, fName2, skip);
+	} else
+	if (mode == "-s") {
+		search<FMindex, String>(fName1, fName2, putHash, bench);
+	} else
+	if (mode == "-r") {
+		recover<FMindex, String>(fName1, fName2);
+	} else
+	if (mode == "-ss") {
+		simpleSearch<String>(fName1, fName2, putHash);
+	} else
+	{
+		usage();
+	}
+} catch (std::exception& e) {
+	printf("ERR %s\n", e.what());
+	return 1;
+}
+

data/vendor/cybozulib/sample/log_smpl.cpp

@@ -0,0 +1,19 @@
+#include <stdio.h>
+#include <cybozu/log.hpp>
+
+int main()
+{
+	cybozu::PutLog(cybozu::LogInfo, "this is a pen1");
+	cybozu::useSyslog(false);
+	cybozu::SetLogUseMsec();
+	cybozu::PutLog(cybozu::LogInfo, "this is a pen2");
+	cybozu::OpenLogFile("test.log");
+	cybozu::PutLog(cybozu::LogInfo, "this is a pen3");
+	cybozu::useSyslog(true);
+	cybozu::PutLog(cybozu::LogInfo, "this is a pen4");
+
+	cybozu::PutLog(cybozu::LogInfo, "AAtest");
+	cybozu::SetLogPriority(cybozu::LogInfo);
+	cybozu::PutLog(cybozu::LogInfo, "AAtest2");
+	cybozu::PutLog(cybozu::LogDebug, "not print");
+}

data/vendor/cybozulib/sample/mecab_smpl.cpp

@@ -0,0 +1,37 @@
+#include <vector>
+#include <stdio.h>
+#include <cybozu/nlp/mecab.hpp>
+#include <cybozu/mmap.hpp>
+
+int main(int argc, char *argv[])
+{
+	argc--, argv++;
+	if (argc == 0) {
+		fprintf(stderr, "mecab_smpl filename\n");
+		return 1;
+	}
+	try {
+		const std::string fileName = argv[0];
+		cybozu::Mmap mmap(fileName);
+		if (mmap.size() > (1 << 30)) {
+			fprintf(stderr, "file is too large %lld\n", (long long)mmap.size());
+			return 1;
+		}
+
+		cybozu::nlp::Mecab mecab;
+		typedef std::vector<std::string> StrVec;
+		StrVec sv;
+		if (mecab.parse(sv, mmap.get(), (int)mmap.size())) {
+			for (size_t i = 0, n = sv.size(); i < n; i++) {
+				printf("%s ", sv[i].c_str());
+			}
+			printf("\n");
+		}
+		return 0;
+	} catch (std::exception& e) {
+		fprintf(stderr, "exception %s\n", e.what());
+	} catch (...) {
+		fprintf(stderr, "unknown exception\n");
+	}
+	return 1;
+}

data/vendor/cybozulib/sample/option2_smpl.cpp

@@ -0,0 +1,68 @@
+/*
+	how to use two step option parser
+*/
+#include <stdio.h>
+#include <cybozu/option.hpp>
+#include <vector>
+
+struct Opt {
+	// common option
+	int x;
+	cybozu::Option opt1;
+
+	// cmd option
+	std::string cmd;
+
+	std::string init_s;
+	double run_d;
+	char status_c;
+	cybozu::Option opt2;
+
+	int parse1(int argc, char *argv[])
+	{
+		opt1.appendOpt(&x, 5, "x", " :value");
+		opt1.appendDelimiter("init");
+		opt1.appendDelimiter("run");
+		opt1.appendDelimiter("status");
+		opt1.appendHelp("h");
+		opt1.setUsage("option2 [opt] (init|run|status)", true);
+
+		if (!opt1.parse(argc, argv)) return false;
+		const int pos = opt1.getNextPositionOfDelimiter();
+		if (pos == 0) return 0;
+		cmd = argv[pos - 1];
+		if (cmd == "init") {
+			opt2.appendOpt(&init_s, "abc", "s", " :string");
+		} else if (cmd == "run") {
+			opt2.appendOpt(&run_d, 1.2, "d", " :double");
+		} else if (cmd == "status") {
+			opt2.appendOpt(&status_c, 'X', "c", " :char");
+		} else {
+			return 0;
+		}
+		opt2.appendHelp("h");
+		return pos;
+	}
+	void parse(int argc, char *argv[])
+	{
+		int pos = parse1(argc, argv);
+		if (pos == 0) {
+			opt1.usage();
+			exit(1);
+		}
+		if (!opt2.parse(argc, argv, pos)) {
+			opt2.usage();
+			exit(1);
+		}
+		puts("common");
+		opt1.put();
+		printf("opt for %s\n", cmd.c_str());
+		opt2.put();
+	}
+};
+
+int main(int argc, char *argv[])
+{
+	Opt opt;
+	opt.parse(argc, argv);
+}

data/vendor/cybozulib/sample/option_smpl.cpp

@@ -0,0 +1,42 @@
+/*
+	how to use
+*/
+#include <stdio.h>
+#include <cybozu/option.hpp>
+#include <vector>
+
+int main(int argc, char *argv[])
+	try
+{
+	int x;
+	bool b;
+	double d;
+	std::string y;
+	std::vector<int> z;
+	std::vector<std::string> w;
+	std::string inName;
+	std::vector<std::string> r;
+	std::vector<std::string> vi;
+	uint64_t u;
+
+	cybozu::Option opt;
+
+	opt.appendOpt(&x, 5, "x", "int");
+	opt.appendBoolOpt(&b, "b", "bool");
+	opt.appendMust(&d, "d", "double");
+	opt.appendMust(&y, "y", "string");
+	opt.appendVec(&z, "z", "int int int ...");
+	opt.appendVec(&w, "w", "str str str ...");
+	opt.appendOpt(&u, 0, "u", "uint64 val");
+	opt.appendParam(&inName, "input-file", "text file");
+	opt.appendParamVec(&vi, "remains", "sss");
+	opt.appendHelp("h");
+
+	if (opt.parse(argc, argv)) {
+		opt.put();
+	} else {
+		opt.usage();
+	}
+} catch (std::exception& e) {
+	printf("ERR %s\n", e.what());
+}

data/vendor/cybozulib/sample/plsi_smpl.cpp

@@ -0,0 +1,207 @@
+/**
+	pLSI(probabilistic latent semantic indexing)
+	@author MITSUNARI Shigeo(@herumi)
+*/
+
+#include <stdio.h>
+#include <map>
+#include <cybozu/file.hpp>
+#include <cybozu/csv.hpp>
+#include <cybozu/nlp/plsi.hpp>
+#include <cybozu/string_operation.hpp>
+#include <cybozu/time.hpp>
+#include <iostream>
+
+void load(cybozu::nlp::Plsi& plsi, const std::string& filepath)
+{
+	cybozu::CsvReader csv(filepath, ' ');
+	std::vector<std::string> line;
+	while (csv.read(line)) {
+		cybozu::nlp::Plsi::ITEM_TYPE item_key = cybozu::atoi(line[0]);
+		size_t size = line.size();
+		if (size < 2) continue;
+		std::map<size_t, bool> map;
+		for (size_t i = 1; i < line.size(); ++i) {
+			cybozu::nlp::Plsi::USER_TYPE user_key = cybozu::atoi(line[i]);
+			map[plsi.get_user_id(user_key)] = true;
+		}
+		plsi.getItem(item_key).set(map);
+	}
+}
+
+void usage()
+{
+	printf("usage: plsi [option] -f [dataset filename]\n");
+	printf("  -k [num] : # of latent classes");
+	printf("  -i [num] : # of iterations");
+	exit(1);
+}
+
+/**
+	@brief Atnd Data
+*/
+struct AtndData {
+	std::string date; // for only event
+	std::string name; // user or event name
+};
+
+/**
+	@brief Atnd Information (Users / Events)
+*/
+struct AtndInfo {
+	typedef std::map<int, AtndData> Int2Data;
+	typedef std::map<std::string, int> Str2Int;
+	Int2Data int2data_;
+	Str2Int name2id_;
+	/**
+		@brief load list of Atnd Users / Events
+		@param[in] name	  filename of list
+		@param[in] isEvent   Is it an event list?
+	*/
+	bool loadList(const std::string& name, bool isEvent)
+	{
+		std::ifstream ifs(name.c_str(), std::ios::binary);
+		if (!ifs) return false;
+		for (;;) {
+			AtndData t;
+			int id;
+			if (!(ifs >> id)) break;
+			if (isEvent) {
+				std::string str;
+				ifs >> str;
+				if (str.empty()) return false;
+				if (str.size() < 6) {
+					fprintf(stderr, "bad format %s\n", str.c_str());
+					return false;
+				}
+				str = str.substr(0, str.size() - 6); // "+09:00"
+				cybozu::Time time(str);
+				time.setTime(time.getTime() + 9	* 3600);
+				time.toString(t.date, "%Y/%m/%d", false);
+			}
+			std::getline(ifs, t.name);
+			cybozu::Trim(t.name);
+			if (!ifs) break;
+			int2data_[id] = t;
+			name2id_[t.name] = id;
+		}
+		return true;
+	}
+	/**
+		@brief load list of Atnd Users / Events. (generates filename from isEvent parameter)
+		@param[in] dir	   directory name where list exists
+		@param[in] isEvent   Is it an event list?
+	*/
+	bool load(const std::string& dir, bool isEvent)
+	{
+		const std::string key = isEvent ? "event" : "user";
+		std::string name;
+		name = dir + "/atnd-" + key + ".txt";
+		if (!loadList(name, isEvent)) {
+			fprintf(stderr, "can't read %s (%d)\n", name.c_str(), isEvent);
+			return false;
+		}
+		return true;
+	}
+};
+
+int main(int argc, char** argv)
+{
+	std::string data_dir = cybozu::GetExePath() + "../sample/data/plsi/";
+
+	int K = 20;
+	int Iter = 100;
+	argc--, argv++;
+	while (argc > 0) {
+		if (argc > 1 && strcmp(*argv, "-d") == 0) {
+			argc--, argv++;
+			data_dir = *argv;
+		} else if (argc > 1 && strcmp(*argv, "-k") == 0) {
+			argc--, argv++;
+			K = cybozu::atoi(*argv);
+		} else if (argc > 1 && strcmp(*argv, "-i") == 0) {
+			argc--, argv++;
+			Iter = cybozu::atoi(*argv);
+		} else {
+			usage();
+		}
+		argc--, argv++;
+	}
+	const std::string name = data_dir + "/atnd-user-matrix.txt";
+
+	cybozu::nlp::Plsi plsi;
+	try {
+		AtndInfo event_master, user_master;
+		event_master.load(data_dir, true);
+		user_master.load(data_dir, false);
+
+		load(plsi, name);
+		plsi.startLearning(K);
+		{
+			puts("learning");
+			double pre_likelihood = -1e30;
+			double beta = 1;
+			for (int i = 0; i < Iter; ++i) {
+				double likelihood = plsi.step();
+				printf("%d : %.3f %.3f %.3f\n", i, beta, likelihood, likelihood - pre_likelihood);
+				if (likelihood - pre_likelihood < 1) {
+					beta *= 0.9;
+					if (beta < 0.01) break;
+				}
+				pre_likelihood = likelihood;
+			}
+		}
+
+		int mode = 0;
+		cybozu::nlp::Plsi::SEARCH_TYPE search_type = cybozu::nlp::Plsi::JOINT;
+
+		for(;;) {
+			std::string st;
+			std::cin >> st;
+			if (st == "") break;
+			if (st == "ui") {
+				mode = 0;
+				printf("user => items\n");
+				continue;
+			}
+			if (st == "ii") {
+				mode = 1;
+				printf("item => items\n");
+				continue;
+			}
+			if (st == "sj") {
+				search_type = cybozu::nlp::Plsi::JOINT;
+				printf("search type: JOINT probability\n");
+				continue;
+			}
+			if (st == "sc") {
+				search_type = cybozu::nlp::Plsi::CONDITIONAL;
+				printf("search type: CONDITIONAL probability\n");
+				continue;
+			}
+			if (st == "sp") {
+				search_type = cybozu::nlp::Plsi::POSTERIOR;
+				printf("search type: POSTERIOR probability\n");
+				continue;
+			}
+
+			cybozu::nlp::TopScore<size_t>::Table tbl;
+			switch(mode) {
+			case 0:
+				tbl = plsi.search_items(cybozu::atoi(st), 10);
+				break;
+			case 1:
+				tbl = plsi.similar_items(cybozu::atoi(st), search_type, 10);
+				break;
+			}
+
+			for (size_t i = 0; i < tbl.size(); i++) {
+				cybozu::nlp::Plsi::ITEM_TYPE key = plsi.get_item_key(tbl[i].idx);
+				printf("%1.3f %d:%s\n", log(tbl[i].score), key, event_master.int2data_[key].name.c_str());
+			}
+		}
+
+	} catch (std::exception& e) {
+		printf("error : %s\n", e.what());
+	}
+}