scws4r 0.1.0

data/ext/scws4r/xdict.c

@@ -0,0 +1,394 @@
+/**
+ * @file xdict.c (dictionary query)
+ * @author Hightman Mar
+ * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim)
+ * $Id$
+ */
+
+#ifdef HAVE_CONFIG_H
+#    include "config.h"
+#endif
+
+#ifdef WIN32
+#    include "config_win32.h"
+#endif
+
+#include "xdict.h"
+#include "xtree.h"
+#include "xdb.h"
+#include "crc32.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#ifndef WIN32
+#    include <sys/param.h>
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+
+/* temp file format for TEXT xdb */
+#if !defined(PATH_MAX) || (PATH_MAX < 1024)
+#    define	XDICT_PATH_MAX	1024
+#else
+#    define	XDICT_PATH_MAX	PATH_MAX
+#endif
+
+#ifdef HAVE_STRTOK_R
+#    define	_strtok_r	strtok_r
+#else
+
+static char *_strtok_r(char *s, char *delim, char **lasts)
+{
+	register char *spanp;
+	register int c, sc;
+	char *tok;
+
+	if (s == NULL && (s = *lasts) == NULL)
+		return NULL;
+
+	/*
+	 * Skip (span) leading delimiters (s += strspn(s, delim), sort of).
+	 */
+cont:
+	c = *s++;
+	for (spanp = (char *) delim; (sc = *spanp++) != 0;)
+	{
+		if (c == sc) goto cont;
+	}
+
+	if (c == 0)
+	{ /* no non-delimiter characters */
+		*lasts = NULL;
+		return NULL;
+	}
+	tok = s - 1;
+
+	/*
+	 * Scan token (scan for delimiters: s += strcspn(s, delim), sort of).
+	 * Note that delim must have one NUL; we stop if we see that, too.
+	 */
+	for (;;)
+	{
+		c = *s++;
+		spanp = (char *) delim;
+		do
+		{
+			if ((sc = *spanp++) == c)
+			{
+				if (c == 0) s = NULL;
+				else s[-1] = '\0';
+				*lasts = s;
+				return tok;
+			}
+		}
+		while (sc != 0);
+	}
+}
+#endif
+
+#ifdef WIN32
+#    include <direct.h>
+
+static void _realpath(const char *src, char *dst)
+{
+	int len = strlen(src);
+	if (strchr(src, ':') != NULL)
+		memcpy(dst, src, len + 1);
+	else
+	{
+		char *ptr;
+		getcwd(dst, XDICT_PATH_MAX - len - 2);
+		ptr = dst + strlen(dst);
+		*ptr++ = '/';
+		memcpy(ptr, src, len + 1);
+	}
+}
+#else
+#    define	_realpath	realpath
+#endif
+
+/* open the text dict */
+static xdict_t _xdict_open_txt(const char *fpath, int mode, unsigned char *ml)
+{
+	xdict_t xd;
+	xtree_t xt;
+	char buf[XDICT_PATH_MAX], tmpfile[XDICT_PATH_MAX];
+	struct stat st1, st2;
+
+	// check the input filepath
+	_realpath(fpath, buf);
+	if (stat(buf, &st1) < 0)
+		return NULL;
+
+	// check dest file & orginal file, compare there mtime
+#ifdef WIN32
+	{
+		char *tmp_ptr;
+		GetTempPath(sizeof(tmpfile) - 20, tmpfile);
+		tmp_ptr = tmpfile + strlen(tmpfile);
+		if (tmp_ptr[-1] == '\\') tmp_ptr--;
+		sprintf(tmp_ptr, "\\scws-%08x.xdb", scws_crc32(buf));
+	}
+#else
+	sprintf(tmpfile, "/tmp/scws-%08x.xdb", scws_crc32(buf));
+#endif
+	if (!stat(tmpfile, &st2) && st2.st_mtime > st1.st_mtime)
+	{
+		xdb_t x;
+		if ((x = xdb_open(tmpfile, 'r')) != NULL)
+		{
+			xd = (xdict_t) malloc(sizeof(xdict_st));
+			memset(xd, 0, sizeof(xdict_st));
+			xd->ref = 1;
+
+			if (mode & SCWS_XDICT_MEM)
+			{
+				/* convert the xdb(disk) -> xtree(memory) */
+				if ((xt = xdb_to_xtree(x, NULL)) != NULL)
+				{
+					xdb_close(x);
+					xd->xdict = (void *) xt;
+					xd->xmode = SCWS_XDICT_MEM;
+					return xd;
+				}
+			}
+			xd->xmode = SCWS_XDICT_XDB;
+			xd->xdict = (void *) x;
+			return xd;
+		}
+	}
+
+	// create xtree
+	if ((xt = xtree_new(0, 0)) == NULL)
+		return NULL;
+	else
+	{
+		int cl, kl;
+		FILE *fp;
+		word_st word, *w;
+		char *key, *part, *last, *delim = " \t\r\n";
+
+		// re-build the xdb file from text file	
+		if ((fp = fopen(buf, "r")) == NULL)
+			return NULL;
+
+		// parse every line
+		word.attr[2] = '\0';
+		while (fgets(buf, sizeof(buf) - 1, fp) != NULL)
+		{
+			// <word>[\t<tf>[\t<idf>[\t<attr>]]]		
+			if (buf[0] == ';' || buf[0] == '#') continue;
+
+			key = _strtok_r(buf, delim, &last);
+			if (key == NULL) continue;
+			kl = strlen(key);
+
+			// init the word
+			do
+			{
+				word.tf = word.idf = 1.0;
+				word.flag = SCWS_WORD_FULL;
+				word.attr[0] = '@';
+				word.attr[1] = '\0';
+
+				if (!(part = _strtok_r(NULL, delim, &last))) break;
+				word.tf = (float) atof(part);
+
+				if (!(part = _strtok_r(NULL, delim, &last))) break;
+				word.idf = (float) atof(part);
+
+				if (part = _strtok_r(NULL, delim, &last))
+				{
+					word.attr[0] = part[0];
+					if (part[1]) word.attr[1] = part[1];
+				}
+			}
+			while (0);
+
+			// save into xtree
+			if ((w = xtree_nget(xt, key, kl, NULL)) == NULL)
+			{
+				w = (word_st *) pmalloc(xt->p, sizeof(word_st));
+				memcpy(w, &word, sizeof(word));
+				xtree_nput(xt, w, sizeof(word), key, kl);
+			}
+			else
+			{
+				w->tf = word.tf;
+				w->idf = word.idf;
+				w->flag |= word.flag;
+				strcpy(w->attr, word.attr);
+			}
+
+			// parse the part	
+			cl = ml[(unsigned char) (key[0])];
+			while (1)
+			{
+				cl += ml[(unsigned char) (key[cl])];
+				if (cl >= kl) break;
+
+				if ((w = xtree_nget(xt, key, cl, NULL)) != NULL)
+					w->flag |= SCWS_WORD_PART;
+				else
+				{
+					w = (word_st *) pmalloc_z(xt->p, sizeof(word_st));
+					w->flag = SCWS_WORD_PART;
+					xtree_nput(xt, w, sizeof(word), key, cl);
+				}
+			}
+		}
+		fclose(fp);
+
+		// optimize the xtree & save to xdb
+		xtree_optimize(xt);
+		unlink(tmpfile);
+		xtree_to_xdb(xt, tmpfile);
+		chmod(tmpfile, 0777);
+
+		// return xtree
+		xd = (xdict_t) malloc(sizeof(xdict_st));
+		memset(xd, 0, sizeof(xdict_st));
+		xd->ref = 1;
+		xd->xdict = (void *) xt;
+		xd->xmode = SCWS_XDICT_MEM;
+		return xd;
+	}
+}
+
+/* setup & open the dict */
+xdict_t xdict_open(const char *fpath, int mode)
+{
+	xdict_t xd;
+	xdb_t x;
+
+	if (!(x = xdb_open(fpath, 'r')))
+		return NULL;
+
+	xd = (xdict_t) malloc(sizeof(xdict_st));
+	memset(xd, 0, sizeof(xdict_st));
+	xd->ref = 1;
+	if (mode & SCWS_XDICT_MEM)
+	{
+		xtree_t xt;
+
+		/* convert the xdb(disk) -> xtree(memory) */
+		if ((xt = xdb_to_xtree(x, NULL)) != NULL)
+		{
+			xdb_close(x);
+			xd->xdict = (void *) xt;
+			xd->xmode = SCWS_XDICT_MEM;
+			return xd;
+		}
+	}
+
+	xd->xmode = SCWS_XDICT_XDB;
+	xd->xdict = (void *) x;
+	return xd;
+}
+
+/* add a dict */
+xdict_t xdict_add(xdict_t xd, const char *fpath, int mode, unsigned char *ml)
+{
+	xdict_t xx;
+
+	xx = (mode & SCWS_XDICT_TXT ? _xdict_open_txt(fpath, mode, ml) : xdict_open(fpath, mode));
+	if (xx != NULL)
+	{
+		xx->next = xd;
+		return xx;
+	}
+	return xd;
+}
+
+/* fork the dict */
+xdict_t xdict_fork(xdict_t xd)
+{
+	xdict_t xx;
+	for (xx = xd; xx != NULL; xx = xx->next)
+	{
+		xx->ref++;
+	}
+	return xd;
+}
+
+/* close the dict */
+void xdict_close(xdict_t xd)
+{
+	xdict_t xx;
+
+	while ((xx = xd) != NULL)
+	{
+		xd = xx->next;
+		xx->ref--;
+		if (xx->ref == 0)
+		{
+			if (xx->xmode == SCWS_XDICT_MEM)
+				xtree_free((xtree_t) xx->xdict);
+			else
+			{
+				xdb_close((xdb_t) xx->xdict);
+			}
+			free(xx);
+		}
+	}
+}
+
+/* query the word */
+#define	_FLAG_BOTH(x)	(((x)->flag & (SCWS_WORD_PART|SCWS_WORD_FULL)) == (SCWS_WORD_PART|SCWS_WORD_FULL))
+#define	_FLAG_FULL(x)	((x)->flag & SCWS_WORD_FULL)
+#define	_FLAG_PART(x)	((x)->flag & SCWS_WORD_PART)
+#define	_FLAG_MALLOC(x)	((x)->flag & SCWS_WORD_MALLOCED)
+
+word_t xdict_query(xdict_t xd, const char *key, int len)
+{
+	word_t value, value2;
+
+	value = value2 = NULL;
+	while (xd != NULL)
+	{
+		if (xd->xmode == SCWS_XDICT_MEM)
+		{
+			/* this is ThreadSafe, recommend. */
+			value = (word_t) xtree_nget((xtree_t) xd->xdict, key, len, NULL);
+		}
+		else
+		{
+			/* the value malloced in lib-XDB. free required */
+			value = (word_t) xdb_nget((xdb_t) xd->xdict, key, len, NULL);
+			if (value != NULL) value->flag |= SCWS_WORD_MALLOCED;
+		}
+		xd = xd->next;
+
+		// check value2
+		if (value != NULL)
+		{
+			if (value2 == NULL)
+			{
+				if (_FLAG_BOTH(value))
+					return value;
+				value2 = value;
+			}
+			else
+			{
+				if (_FLAG_FULL(value2) && _FLAG_PART(value))
+				{
+					value2->flag |= SCWS_WORD_PART;
+					if (_FLAG_MALLOC(value))
+						free(value);
+					return value2;
+				}
+				if (_FLAG_FULL(value) && _FLAG_PART(value2))
+				{
+					value->flag |= SCWS_WORD_PART;
+					if (_FLAG_MALLOC(value2))
+						free(value2);
+					return value;
+				}
+				if (_FLAG_MALLOC(value))
+					free(value);
+			}
+		}
+	}
+	return value2;
+}

data/ext/scws4r/xdict.h

@@ -0,0 +1,73 @@
+/**
+ * @file xdict (dictionary)
+ * @author Hightman Mar
+ * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim)
+ * $Id$
+ */
+
+#ifndef	_SCWS_XDICT_20070528_H_
+#define	_SCWS_XDICT_20070528_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* constant var define */
+#define	SCWS_WORD_FULL		0x01	// 多字: 整词
+#define	SCWS_WORD_PART		0x02	// 多字: 前词段
+#define	SCWS_WORD_USED		0x04	// 多字: 已使用
+#define	SCWS_WORD_RULE		0x08	// 多字: 自动识别的
+#define	SCWS_WORD_LONG		0x10	// 多字: 短词组成的长词
+
+#define	SCWS_WORD_MALLOCED	0x80	// xdict_query 结果必须调用 free
+
+#define	SCWS_ZFLAG_PUT		0x02	// 单字: 已使用
+#define	SCWS_ZFLAG_N2		0x04	// 单字: 双字名词头
+#define	SCWS_ZFLAG_NR2		0x08	// 单字: 词头且为双字人名
+#define	SCWS_ZFLAG_WHEAD	0x10	// 单字: 词头
+#define	SCWS_ZFLAG_WPART	0x20	// 单字: 词尾或词中
+#define	SCWS_ZFLAG_ENGLISH	0x40	// 单字: 夹在中间的英文
+#define SCWS_ZFLAG_SYMBOL   0x80    // 单字: 符号系列
+#define	SCWS_XDICT_PRIME	0x3ffd	// 词典结构树数：16381
+
+/* xdict open mode */
+#define	SCWS_XDICT_XDB		1
+#define	SCWS_XDICT_MEM		2
+#define	SCWS_XDICT_TXT		4		// ...
+#define	SCWS_XDICT_SET		4096	// set flag.
+
+/* data structure for word(12bytes) */
+typedef struct scws_word
+{
+	float tf;
+	float idf;
+	unsigned char flag;
+	char attr[3];
+}	word_st, *word_t;
+
+typedef struct scws_xdict
+{
+	void *xdict;
+	int xmode;
+	int ref;	// hightman.20130110: refcount (zero to really free/close)
+	struct scws_xdict *next;
+}	xdict_st, *xdict_t;
+
+/* pub function (api) */
+xdict_t xdict_open(const char *fpath, int mode);
+void xdict_close(xdict_t xd);
+
+/* fork xdict */
+xdict_t xdict_fork(xdict_t xd);
+
+/* add a new dict file into xd, succ: 0, error: -1, Mblen only used for XDICT_TXT */
+xdict_t xdict_add(xdict_t xd, const char *fpath, int mode, unsigned char *ml);
+
+/* NOW this is ThreadSafe function */
+word_t xdict_query(xdict_t xd, const char *key, int len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif