wapiti 0.0.1

data/ext/wapiti/pattern.h

@@ -0,0 +1,56 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef pattern_h
+#define pattern_h
+
+#include <stdbool.h>
+
+#include "sequence.h"
+
+typedef struct pat_s pat_t;
+typedef struct pat_item_s pat_item_t;
+struct pat_s {
+	char *src;
+	int   ntoks;
+	int   nitems;
+	struct pat_item_s {
+		char  type;
+		bool  caps;
+		char *value;
+		bool  absolute;
+		int   offset;
+		int   column;
+	} items[];
+};
+
+pat_t *pat_comp(char *p);
+char *pat_exec(const pat_t *pat, const tok_t *tok, int at);
+void pat_free(pat_t *pat);
+
+#endif
+

data/ext/wapiti/progress.c

@@ -0,0 +1,167 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <signal.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <unistd.h>
+#include <sys/times.h>
+#include <sys/resource.h>
+
+#include "wapiti.h"
+#include "decoder.h"
+#include "model.h"
+#include "options.h"
+#include "progress.h"
+#include "tools.h"
+
+/*******************************************************************************
+ * User interaction during training
+ *
+ *   Handle progress reporting during training and clean early stoping. Trainers
+ *   have to call uit_progress at the end of each iterations, this will display
+ *   various informations for the user.
+ *   Timing is also done here, an iteration is assumed to take all the time
+ *   between to call to the progress function and evualtion on the devel data
+ *   are included.
+ *
+ *   This module setup a signal handler for SIGINT. If this signal is catched,
+ *   the uit_stop global variable to inform the trainer that it have to stop as
+ *   early as possible, discarding the recent computations if they cannot be
+ *   integrated very quickly. They must leave the model in a clean state. Any
+ *   further signal will terminate the program. So it's simple :
+ *     - 1 signal mean "I can wait a little so try to stop as soon as possible
+ *         but leave me a working model"
+ *     - 2 signal mean "Stop immediatly what you are doing, I can't wait and
+ *         don't care about getting a working model"
+ ******************************************************************************/
+
+/* uit_stop:
+ *   This value is set to true when the user request the trainer to stop. In
+ *   this case, the trainer have to stop as soon as possible in a clean state,
+ *   discarding the lasts computations if it cannot integrate them quickly.
+ */
+bool uit_stop = false;
+
+/* uit_signal:
+ *   Signal handler to catch interupt signal. When a signal is received, the
+ *   trainer is aksed to stop as soon as possible leaving the model in a clean
+ *   state. We don't reinstall the handler so if user send a second interupt
+ *   signal, the program will stop imediatly. (to cope with BSD system, we even
+ *   reinstall explicitly the default handler)
+ */
+static void uit_signal(int sig) {
+	signal(sig, SIG_DFL);
+	uit_stop = true;
+}
+
+/* uit_setup:
+ *   Install the signal handler for clean early stop from the user if possible
+ *   and start the timer.
+ */
+void uit_setup(mdl_t *mdl) {
+	uit_stop = false;
+	if (signal(SIGINT, uit_signal) == SIG_ERR)
+		warning("failed to set signal handler, no clean early stop");
+	times(&mdl->timer);
+	if (mdl->opt->stopwin != 0)
+		mdl->werr = wapiti_xmalloc(sizeof(double) * mdl->opt->stopwin);
+	mdl->wcnt = mdl->wpos = 0;
+}
+
+/* uit_cleanup:
+ *   Remove the signal handler restoring the defaul behavior in case of
+ *   interrupt.
+ */
+void uit_cleanup(mdl_t *mdl) {
+	unused(mdl);
+	if (mdl->opt->stopwin != 0) {
+		free(mdl->werr);
+		mdl->werr = NULL;
+	}
+	signal(SIGINT, SIG_DFL);
+}
+
+/* uit_progress:
+ *   Display a progress repport to the user consisting of some informations
+ *   provided by the trainer: iteration count and objective function value, and
+ *   some informations computed here on the current model performances.
+ *   This function return true if the trainer have to keep training the model
+ *   and false if he must stop, so this is were we will implement the trainer
+ *   independant stoping criterion.
+ */
+bool uit_progress(mdl_t *mdl, int it, double obj) {
+	// First we just compute the error rate on devel or train data
+	double te, se;
+	tag_eval(mdl, &te, &se);
+	// Next, we compute the number of active features
+	size_t act = 0;
+	for (size_t f = 0; f < mdl->nftr; f++)
+		if (mdl->theta[f] != 0.0)
+			act++;
+	// Compute timings. As some training algorithms are multi-threaded, we
+	// cannot use ansi/c function and must rely on posix one to sum time
+	// spent in main thread and in child ones.
+	tms_t now; times(&now);
+	double tm = (now.tms_utime  - mdl->timer.tms_utime )
+	          + (now.tms_cutime - mdl->timer.tms_cutime);
+	tm /= sysconf(_SC_CLK_TCK);
+	mdl->total += tm;
+	mdl->timer  = now;
+	// And display progress report
+	info("  [%4d]", it);
+	info(obj >= 0.0 ? " obj=%-10.2f" : " obj=NA", obj);
+	info(" act=%-8zu", act);
+	info(" err=%5.2f%%/%5.2f%%", te, se);
+	info(" time=%.2fs/%.2fs", tm, mdl->total);
+	info("\n");
+	// If requested, check the error rate stoping criterion. We check if the
+	// error rate is stable enought over a few iterations.
+	bool res = true;
+	if (mdl->opt->stopwin != 0) {
+		mdl->werr[mdl->wpos] = te;
+		mdl->wpos = (mdl->wpos + 1) % mdl->opt->stopwin;
+		mdl->wcnt++;
+		if (mdl->wcnt >= mdl->opt->stopwin) {
+			double emin = 200.0, emax = -100.0;
+			for (int i = 0; i < mdl->opt->stopwin; i++) {
+				emin = min(emin, mdl->werr[i]);
+				emax = max(emax, mdl->werr[i]);
+			}
+			if (emax - emin < mdl->opt->stopeps)
+				res = false;
+		}
+	}
+	// And return
+	if (uit_stop)
+		return false;
+	return res;
+}
+
+

data/ext/wapiti/progress.h

@@ -0,0 +1,43 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef progress_h
+#define progress_h
+
+#include <stdbool.h>
+
+#include "wapiti.h"
+#include "model.h"
+
+extern bool uit_stop;
+
+void uit_setup(mdl_t *mdl);
+void uit_cleanup(mdl_t *mdl);
+bool uit_progress(mdl_t *mdl, int it, double obj);
+
+#endif
+

data/ext/wapiti/quark.c

@@ -0,0 +1,272 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "quark.h"
+#include "tools.h"
+
+/******************************************************************************
+ * Map object
+ *
+ *   Implement quark object for mapping strings to identifiers through crit-bit
+ *   tree (also known as PATRICIA tries). In fact it only store a compressed
+ *   version of the trie to reduce memory footprint. The special trick of using
+ *   the last bit of the reference to differenciate between nodes and leafs come
+ *   from Daniel J. Bernstein implementation of crit-bit tree that can be found
+ *   on his web site.
+ *   [1] Morrison, Donald R. ; PATRICIA-Practical Algorithm To Retrieve
+ *   Information Coded in Alphanumeric, Journal of the ACM 15 (4): pp. 514--534,
+ *   1968. DOI:10.1145/321479.321481
+ *
+ *   This code is copyright 2002-2011 Thomas Lavergne and licenced under the BSD
+ *   Licence like the remaining of Wapiti.
+ ******************************************************************************/
+
+typedef struct node_s node_t;
+typedef struct leaf_s leaf_t;
+struct qrk_s {
+	struct node_s {
+		node_t   *child[2];
+		uint32_t  pos;
+		uint8_t   byte;
+	} *root;
+	struct leaf_s {
+		uint64_t  id;
+		char      key[];
+	} **leafs;
+	bool     lock;
+	uint64_t count;
+	uint64_t size;
+};
+
+#define qrk_none ((uint64_t)-1)
+
+#define qrk_lf2nd(lf)  ((node_t *)((intptr_t)(lf) |  1))
+#define qrk_nd2lf(nd)  ((leaf_t *)((intptr_t)(nd) & ~1))
+#define qrk_isleaf(nd) ((intptr_t)(nd) & 1)
+
+/* qrk_new:
+ *   This initialize the object for holding a new empty trie, with some pre-
+ *   allocations. The returned object must be freed with a call to qrk_free when
+ *   not needed anymore.
+ */
+qrk_t *qrk_new(void) {
+	const uint64_t size = 128;
+	qrk_t *qrk = wapiti_xmalloc(sizeof(qrk_t));
+	qrk->root  = NULL;
+	qrk->count = 0;
+	qrk->lock  = false;
+	qrk->size  = size;
+	qrk->leafs = wapiti_xmalloc(sizeof(leaf_t) * size);
+	return qrk;
+}
+
+/* qrk_free:
+ *   Release all the memory used by a qrk_t object allocated with qrk_new. This
+ *   will release all key string stored internally so all key returned by
+ *   qrk_unmap become invalid and must not be used anymore.
+ */
+void qrk_free(qrk_t *qrk) {
+	const size_t stkmax = 1024;
+	if (qrk->count != 0) {
+		node_t *stk[stkmax];
+		int cnt = 0;
+		stk[cnt++] = qrk->root;
+		while (cnt != 0) {
+			node_t *nd = stk[--cnt];
+			if (qrk_isleaf(nd)) {
+				free(qrk_nd2lf(nd));
+				continue;
+			}
+			stk[cnt++] = nd->child[0];
+			stk[cnt++] = nd->child[1];
+			free(nd);
+		}
+	}
+	free(qrk->leafs);
+	free(qrk);
+}
+
+/* qrk_insert:
+ *   Map a key to a uniq identifier. If the key already exist in the map, return
+ *   its identifier, else allocate a new identifier and insert the new (key,id)
+ *   pair inside the quark. This function is not thread safe and should not be
+ *   called on the same map from different thread without locking.
+ */
+size_t qrk_str2id(qrk_t *qrk, const char *key) {
+	const uint8_t *raw = (void *)key;
+	const size_t   len = strlen(key);
+	// We first take care of the empty trie case so later we can safely
+	// assume that the trie is well formed and so there is no NULL pointers
+	// in it.
+	if (qrk->count == 0) {
+		if (qrk->lock == true)
+			return none;
+		const size_t size = sizeof(char) * (len + 1);
+		leaf_t *lf = wapiti_xmalloc(sizeof(leaf_t) + size);
+		memcpy(lf->key, key, size);
+		lf->id = 0;
+		qrk->root = qrk_lf2nd(lf);
+		qrk->leafs[0] = lf;
+		qrk->count = 1;
+		return 0;
+	}
+	// If the trie is not empty, we first go down the trie to the leaf like
+	// if we are searching for the key. When at leaf there is two case,
+	// either we have found our key or we have found another key with all
+	// its critical bit identical to our one. So we search for the first
+	// differing bit between them to know where we have to add the new node.
+	const node_t *nd = qrk->root;
+	while (!qrk_isleaf(nd)) {
+		const uint8_t chr = nd->pos < len ? raw[nd->pos] : 0;
+		const int side = ((chr | nd->byte) + 1) >> 8;
+		nd = nd->child[side];
+	}
+	const char *bst = qrk_nd2lf(nd)->key;
+	size_t pos;
+	for (pos = 0; pos < len; pos++)
+		if (key[pos] != bst[pos])
+			break;
+	uint8_t byte;
+	if (pos != len)
+		byte = key[pos] ^ bst[pos];
+	else if (bst[pos] != '\0')
+		byte = bst[pos];
+	else
+		return qrk_nd2lf(nd)->id;
+	if (qrk->lock == true)
+		return none;
+	// Now we known the two key are different and we know in which byte. It
+	// remain to build the mask for the new critical bit and build the new
+	// internal node and leaf.
+	while (byte & (byte - 1))
+		byte &= byte - 1;
+	byte ^= 255;
+	const uint8_t chr = bst[pos];
+	const int side = ((chr | byte) + 1) >> 8;
+	const size_t size = sizeof(char) * (len + 1);
+	node_t *nx = wapiti_xmalloc(sizeof(node_t));
+	leaf_t *lf = wapiti_xmalloc(sizeof(leaf_t) + size);
+	memcpy(lf->key, key, size);
+	lf->id   = qrk->count++;
+	nx->pos  = pos;
+	nx->byte = byte;
+	nx->child[1 - side] = qrk_lf2nd(lf);
+	if (lf->id == qrk->size) {
+		qrk->size *= 1.4;
+		const size_t size = sizeof(leaf_t *) * qrk->size;
+		qrk->leafs = wapiti_xrealloc(qrk->leafs, size);
+	}
+	qrk->leafs[lf->id] = lf;
+	// And last thing to do: inserting the new node in the trie. We have to
+	// walk down the trie again as we have to keep the ordering of nodes. So
+	// we search for the good position to insert it.
+	node_t **trg = &qrk->root;
+	while (true) {
+		node_t *nd = *trg;
+		if (qrk_isleaf(nd) || nd->pos > pos)
+			break;
+		if (nd->pos == pos && nd->byte > byte)
+			break;
+		const uint8_t chr = nd->pos < len ? raw[nd->pos] : 0;
+		const int side = ((chr | nd->byte) + 1) >> 8;
+		trg = &nd->child[side];
+	}
+	nx->child[side] = *trg;
+	*trg = nx;
+	return lf->id;
+}
+
+/* qrk_id2str:
+ *    Retrieve the key associated to an identifier. The key is returned as a
+ *    constant string that should not be modified or freed by the caller, it is
+ *    a pointer to the internal copy of the key kept by the map object and
+ *    remain valid only for the life time of the quark, a call to qrk_free will
+ *    make this pointer invalid.
+ */
+const char *qrk_id2str(const qrk_t *qrk, size_t id) {
+	if (id >= qrk->count)
+		fatal("invalid identifier");
+	return qrk->leafs[id]->key;
+}
+
+/* qrk_save:
+ *   Save list of keys present in the map object in the id order to the given
+ *   file. We put one key per line so, if no key contains a new line, the line
+ *   number correspond to the id.
+ */
+void qrk_save(const qrk_t *qrk, FILE *file) {
+	if (fprintf(file, "#qrk#%zu\n", (size_t)qrk->count) < 0)
+		pfatal("cannot write to file");
+	if (qrk->count == 0)
+		return;
+	for (uint64_t n = 0; n < qrk->count; n++)
+		ns_writestr(file, qrk->leafs[n]->key);
+}
+
+/* qrk_load:
+ *   Load a list of key from the given file and add them to the map. Each lines
+ *   of the file is taken as a single key and mapped to the next available id if
+ *   not already present. If all keys are single lines and the given map is
+ *   initilay empty, this will load a map exactly as saved by qrk_save.
+ */
+void qrk_load(qrk_t *qrk, FILE *file) {
+	size_t cnt = 0;
+	if (fscanf(file, "#qrk#%zu\n", &cnt) != 1) {
+		if (ferror(file) != 0)
+			pfatal("cannot read from file");
+		pfatal("invalid format");
+	}
+	for (size_t n = 0; n < cnt; ++n) {
+		char *str = ns_readstr(file);
+		qrk_str2id(qrk, str);
+		free(str);
+	}
+}
+
+/* qrk_count:
+ *   Return the number of mappings stored in the quark.
+ */
+size_t qrk_count(const qrk_t *qrk) {
+	return qrk->count;
+}
+
+/* qrk_lock:
+ *   Set the lock value of the quark and return the old one.
+ */
+bool qrk_lock(qrk_t *qrk, bool lock) {
+	bool old = qrk->lock;
+	qrk->lock = lock;
+	return old;
+}
+
+